cctally 1.61.0 → 1.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/_cctally_cache.py +156 -6
- package/bin/_cctally_config.py +26 -0
- package/bin/_cctally_dashboard.py +670 -162
- package/bin/_cctally_db.py +27 -1
- package/bin/_cctally_parser.py +15 -0
- package/bin/_cctally_statusline.py +9 -1
- package/bin/_cctally_tui.py +74 -9
- package/bin/_cctally_weekrefs.py +9 -0
- package/bin/_lib_aggregators.py +98 -71
- package/bin/_lib_cache_report.py +455 -67
- package/bin/_lib_pricing.py +31 -4
- package/bin/_lib_snapshot_cache.py +869 -60
- package/bin/_lib_statusline.py +25 -13
- package/package.json +1 -1
|
@@ -317,16 +317,21 @@ from _lib_blocks import _group_entries_into_blocks
|
|
|
317
317
|
from _cctally_config import save_config, _load_config_unlocked
|
|
318
318
|
from _cctally_db import _render_migration_error_banner
|
|
319
319
|
from _cctally_cache import (
|
|
320
|
-
get_entries, iter_entries, open_cache_db, sync_cache,
|
|
320
|
+
get_entries, iter_entries, iter_entries_with_id, open_cache_db, sync_cache,
|
|
321
321
|
_prune_orphaned_cache_entries,
|
|
322
322
|
)
|
|
323
323
|
from _lib_snapshot_cache import (
|
|
324
324
|
build_cached_group_a,
|
|
325
325
|
bump_generation,
|
|
326
|
+
cached_bugk_segment,
|
|
327
|
+
reset_bugk_segment_state,
|
|
328
|
+
reset_cache_report_state,
|
|
326
329
|
reset_group_a_state,
|
|
327
330
|
reset_projects_env_state,
|
|
328
331
|
reset_session_cache_state,
|
|
329
332
|
reset_weekref_cost_state,
|
|
333
|
+
BugKSegment,
|
|
334
|
+
_bugk_key,
|
|
330
335
|
_max_id as _snapshot_max_id,
|
|
331
336
|
_reset_sig as _snapshot_reset_sig,
|
|
332
337
|
)
|
|
@@ -420,17 +425,25 @@ def _dashboard_self_heal_orphans(*, skip_sync):
|
|
|
420
425
|
# #269 M3.2 (spec §6): the shared per-weekref immutable-cost cache
|
|
421
426
|
# (B1 trend/weekly-history + B3 forecast) rides the SAME prune-site
|
|
422
427
|
# clear — a non-max deletion the reconcile's max-id regression check
|
|
423
|
-
# can't catch.
|
|
424
|
-
# so there is no reset_cache_report_state() to call here.)
|
|
428
|
+
# can't catch.
|
|
425
429
|
reset_weekref_cost_state()
|
|
430
|
+
# #272: the per-day cache-report cache rides the same prune-site
|
|
431
|
+
# clear — a non-max deletion inside a closed day the reconcile's
|
|
432
|
+
# max-id / mutation-seq regression check can't catch.
|
|
433
|
+
reset_cache_report_state()
|
|
426
434
|
# #269 M4.5 (spec §14 Win 2): the projects-envelope per-(project,
|
|
427
435
|
# week) cache rides the same prune-site clear for the same reason.
|
|
428
436
|
reset_projects_env_state()
|
|
437
|
+
# #271 M3 (spec §18): the Bug-K pre-credit segment cache rides the
|
|
438
|
+
# same prune-site clear — a non-max deletion inside a closed
|
|
439
|
+
# pre-credit window the reconcile's max-id regression check can't
|
|
440
|
+
# catch.
|
|
441
|
+
reset_bugk_segment_state()
|
|
429
442
|
# #269 (final reviewer): the per-(project, week) cache clear above is
|
|
430
443
|
# NOT sufficient on its own. `_build_projects_envelope` consults the
|
|
431
444
|
# whole-envelope memo `_PROJECTS_ENV_MEMO` FIRST — and its memo_key is
|
|
432
|
-
# `(max_id, max_wus_id, cw_key, weeks_back)`, which
|
|
433
|
-
# generation counter. A prune that deletes only NON-max
|
|
445
|
+
# `(max_id, max_wus_id, entry_mutation_seq, cw_key, weeks_back)`, which
|
|
446
|
+
# carries NO generation counter. A prune that deletes only NON-max
|
|
434
447
|
# `session_entries` rows leaves `max_id` unchanged, so the memo_key
|
|
435
448
|
# still matches and the memo would stale-serve the pre-prune envelope
|
|
436
449
|
# (still showing the deleted project + its cost) before the fresh
|
|
@@ -2071,6 +2084,77 @@ def _cache_report_load_kernel():
|
|
|
2071
2084
|
return sys.modules["cctally"]._load_sibling("_lib_cache_report")
|
|
2072
2085
|
|
|
2073
2086
|
|
|
2087
|
+
def _cache_report_needed_closed_dates(since, now_utc, bucket_tz):
|
|
2088
|
+
"""The CLOSED display-tz dates a full-window cache-report build needs (#272 §6).
|
|
2089
|
+
|
|
2090
|
+
Every display-tz calendar date overlapping ``[since, now_utc]`` EXCEPT the
|
|
2091
|
+
current open day (Codex-2: a ``since``-straddling window yields up to
|
|
2092
|
+
``window_days + 1`` display dates, and classification runs over the full
|
|
2093
|
+
set BEFORE the ``days`` cap). Returned sorted ascending. The ``have_all``
|
|
2094
|
+
gate requires every one of these to be cached before serving the warm
|
|
2095
|
+
(today-only-fetch) path. A genuinely activity-free calendar day produces
|
|
2096
|
+
no per-day fold row, so the cold/miss populate branch stores an
|
|
2097
|
+
``is_empty`` sentinel (``empty_cached_day``) for every needed closed date
|
|
2098
|
+
the fold didn't emit — registering the quiet day as a cache HIT (the same
|
|
2099
|
+
computed-empty registration ``projects_env_week_put`` does for an empty
|
|
2100
|
+
week). That keeps ``have_all`` reachable, so a weekend/vacation gap day
|
|
2101
|
+
does NOT permanently defeat the warm path; the sentinel is skipped in the
|
|
2102
|
+
restitch, so the output stays byte-identical to from-scratch.
|
|
2103
|
+
"""
|
|
2104
|
+
today_key = now_utc.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
2105
|
+
d = since.astimezone(bucket_tz).date()
|
|
2106
|
+
end_date = now_utc.astimezone(bucket_tz).date()
|
|
2107
|
+
out: list = []
|
|
2108
|
+
while d <= end_date:
|
|
2109
|
+
key = d.strftime("%Y-%m-%d")
|
|
2110
|
+
if key != today_key:
|
|
2111
|
+
out.append(key)
|
|
2112
|
+
d += dt.timedelta(days=1)
|
|
2113
|
+
return out
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _day_start(date_iso, bucket_tz):
|
|
2117
|
+
"""The aware-UTC instant of ``date_iso``'s local midnight in ``bucket_tz`` (#272 §6).
|
|
2118
|
+
|
|
2119
|
+
Lower-bounds the warm-path current-day fetch so it returns exactly the
|
|
2120
|
+
entries ``_aggregate_cache_by_day`` buckets to ``date_iso`` (which buckets
|
|
2121
|
+
via ``entry.timestamp.astimezone(bucket_tz)``). DST-correct: attaching the
|
|
2122
|
+
zone to the naive local-midnight then converting to UTC lets ``ZoneInfo``
|
|
2123
|
+
pick the right offset for that wall time.
|
|
2124
|
+
"""
|
|
2125
|
+
naive = dt.datetime.strptime(date_iso, "%Y-%m-%d")
|
|
2126
|
+
return naive.replace(tzinfo=bucket_tz).astimezone(dt.timezone.utc)
|
|
2127
|
+
|
|
2128
|
+
|
|
2129
|
+
def _cache_report_empty(
|
|
2130
|
+
*, today_iso, window_days, anomaly_threshold_pp, anomaly_window_days,
|
|
2131
|
+
):
|
|
2132
|
+
"""The empty (no in-window entries) ``CacheReportSnapshot`` — factored so the
|
|
2133
|
+
warm and cold builder paths share one ``is_empty`` return (#272 §6)."""
|
|
2134
|
+
empty_today = CacheReportTodaySpotlight(
|
|
2135
|
+
date=today_iso,
|
|
2136
|
+
cache_hit_percent=0.0,
|
|
2137
|
+
baseline_median_percent=None,
|
|
2138
|
+
delta_pp=None,
|
|
2139
|
+
net_usd=0.0, saved_usd=0.0, wasted_usd=0.0,
|
|
2140
|
+
anomaly_triggered=False,
|
|
2141
|
+
anomaly_reasons=(),
|
|
2142
|
+
baseline_daily_row_count=0,
|
|
2143
|
+
)
|
|
2144
|
+
return CacheReportSnapshot(
|
|
2145
|
+
window_days=window_days,
|
|
2146
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2147
|
+
anomaly_window_days=anomaly_window_days,
|
|
2148
|
+
today=empty_today,
|
|
2149
|
+
days=(), by_project=(), by_model=(),
|
|
2150
|
+
seven_day_net_usd=0.0,
|
|
2151
|
+
seven_day_anomaly_count=0,
|
|
2152
|
+
fourteen_day_counterfactual_usd=0.0,
|
|
2153
|
+
fourteen_day_efficiency_ratio=0.0,
|
|
2154
|
+
is_empty=True,
|
|
2155
|
+
)
|
|
2156
|
+
|
|
2157
|
+
|
|
2074
2158
|
def build_cache_report_snapshot(
|
|
2075
2159
|
*,
|
|
2076
2160
|
now_utc: dt.datetime,
|
|
@@ -2078,91 +2162,167 @@ def build_cache_report_snapshot(
|
|
|
2078
2162
|
anomaly_window_days: int,
|
|
2079
2163
|
display_tz: "ZoneInfo | None",
|
|
2080
2164
|
skip_sync: bool = False,
|
|
2165
|
+
use_cache_report_cache: bool = False,
|
|
2081
2166
|
) -> CacheReportSnapshot:
|
|
2082
2167
|
"""Build the ``cache_report`` envelope field from the session-entry cache.
|
|
2083
2168
|
|
|
2084
2169
|
Pulls entries via ``get_claude_session_entries`` (uses the cache when
|
|
2085
2170
|
warm, falls back to direct-JSONL parse on cache miss / lock
|
|
2086
|
-
contention — same chain the CLI uses).
|
|
2087
|
-
|
|
2088
|
-
|
|
2171
|
+
contention — same chain the CLI uses). Aggregates per closed day via
|
|
2172
|
+
``_lib_cache_report.build_cached_days`` (served from the per-day cache
|
|
2173
|
+
when ``use_cache_report_cache`` and reconciled), reconstructs fresh
|
|
2174
|
+
rows (``reconstruct_cache_row``), runs the cross-row
|
|
2175
|
+
``classify_and_summarize`` + ``combine_day_project_partials`` restitch,
|
|
2176
|
+
and shapes the result into a frozen ``CacheReportSnapshot``.
|
|
2089
2177
|
|
|
2090
2178
|
``window_days`` is hardcoded at 14 in v1 (spec §6.1 hardcodes
|
|
2091
2179
|
``anomaly_window_days`` too; ``anomaly_threshold_pp`` is the only
|
|
2092
2180
|
user-configurable knob). F10 from spec §10 tracks making the window
|
|
2093
2181
|
configurable, plus the UI-copy work it'd require.
|
|
2182
|
+
|
|
2183
|
+
**Per-day cache (#272 §6).** With ``use_cache_report_cache=True`` (set by
|
|
2184
|
+
the dashboard sync thread AFTER ``reconcile_cache_report_cache`` succeeds),
|
|
2185
|
+
the CLOSED days are served from the immutable per-day cache and only the
|
|
2186
|
+
current (open) day is fetched + folded fresh — the warm-tick win. The
|
|
2187
|
+
reconcile (in ``_tui_build_snapshot``) owns invalidation; this builder only
|
|
2188
|
+
reads/populates the cache. With the flag OFF (default / safe fallback), the
|
|
2189
|
+
full window is fetched every tick and the output is byte-identical to the
|
|
2190
|
+
cached path (same restructured assembly, cache bypassed).
|
|
2094
2191
|
"""
|
|
2095
2192
|
crk = _cache_report_load_kernel()
|
|
2096
2193
|
cctally_ns = sys.modules["cctally"]
|
|
2194
|
+
_sc = sys.modules["_lib_snapshot_cache"]
|
|
2097
2195
|
|
|
2098
2196
|
window_days = CACHE_REPORT_WINDOW_DAYS # v1: hardcoded per spec §6.1.
|
|
2099
2197
|
since = now_utc - dt.timedelta(days=window_days)
|
|
2198
|
+
pricing = cctally_ns.CLAUDE_MODEL_PRICING
|
|
2100
2199
|
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2200
|
+
# Cache mechanics key on the BUCKETING tz (``_resolve_bucket_tz`` — the same
|
|
2201
|
+
# tz ``_aggregate_cache_by_day`` buckets by), so ``today_key`` / the closed-
|
|
2202
|
+
# day keys line up with ``CacheRow.date``. The spotlight's ``today_iso``
|
|
2203
|
+
# keeps its own UTC-fallback derivation unchanged (byte-identity); the two
|
|
2204
|
+
# agree whenever ``display_tz`` is set — always, on the dashboard.
|
|
2205
|
+
bucket_tz = crk._resolve_bucket_tz(display_tz)
|
|
2206
|
+
today_key = now_utc.astimezone(bucket_tz).strftime("%Y-%m-%d")
|
|
2105
2207
|
today_iso = now_utc.astimezone(
|
|
2106
2208
|
display_tz if display_tz is not None else dt.timezone.utc
|
|
2107
2209
|
).strftime("%Y-%m-%d")
|
|
2108
2210
|
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2211
|
+
def _wrap_day_entries(raw):
|
|
2212
|
+
# Day-mode kernel expects entries with a ``usage`` dict (matches
|
|
2213
|
+
# ``UsageEntry``); ``get_claude_session_entries`` returns flat
|
|
2214
|
+
# ``_JoinedClaudeEntry`` objects. SimpleNamespace keeps the wrapper
|
|
2215
|
+
# pure-Python and avoids a new dataclass type just for the bridge.
|
|
2216
|
+
from types import SimpleNamespace as _NS
|
|
2217
|
+
return [
|
|
2218
|
+
_NS(
|
|
2219
|
+
timestamp=e.timestamp,
|
|
2220
|
+
model=e.model,
|
|
2221
|
+
cost_usd=e.cost_usd,
|
|
2222
|
+
usage={
|
|
2223
|
+
"input_tokens": e.input_tokens,
|
|
2224
|
+
"output_tokens": e.output_tokens,
|
|
2225
|
+
"cache_creation_input_tokens": e.cache_creation_tokens,
|
|
2226
|
+
"cache_read_input_tokens": e.cache_read_tokens,
|
|
2227
|
+
},
|
|
2228
|
+
)
|
|
2229
|
+
for e in raw
|
|
2230
|
+
]
|
|
2231
|
+
|
|
2232
|
+
# Which CLOSED display-tz dates the window needs (every date overlapping
|
|
2233
|
+
# [since, now] except the current open day). If the cache holds them ALL,
|
|
2234
|
+
# fetch only the current day (the warm win); else one full-window fetch
|
|
2235
|
+
# (re)populates the closed days + recomputes today.
|
|
2236
|
+
needed_closed = _cache_report_needed_closed_dates(since, now_utc, bucket_tz)
|
|
2237
|
+
have_all = use_cache_report_cache and all(
|
|
2238
|
+
_sc.cache_report_day_get(d) is not None for d in needed_closed
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
cached_days: dict = {} # date_key -> CachedCacheReportDay
|
|
2242
|
+
if have_all:
|
|
2243
|
+
# Steady-state warm: recompute ONLY the open bucket; reuse cached closed.
|
|
2244
|
+
raw_today = list(get_claude_session_entries(
|
|
2245
|
+
_day_start(today_key, bucket_tz), now_utc,
|
|
2246
|
+
project=None, skip_sync=skip_sync,
|
|
2247
|
+
))
|
|
2248
|
+
built = crk.build_cached_days(
|
|
2249
|
+
_wrap_day_entries(raw_today), raw_today,
|
|
2250
|
+
display_tz=display_tz, pricing=pricing,
|
|
2251
|
+
cost_calculator=_calculate_entry_cost,
|
|
2252
|
+
)
|
|
2253
|
+
for d in needed_closed:
|
|
2254
|
+
cached_days[d] = _sc.cache_report_day_get(d)
|
|
2255
|
+
cached_days.update(built) # the fresh current day (today only)
|
|
2256
|
+
else:
|
|
2257
|
+
# Cold / partial-miss / post-eviction: pay one full-window fetch, then
|
|
2258
|
+
# (re)compute + store every closed day. Runs on the rare stale tick.
|
|
2259
|
+
raw = list(get_claude_session_entries(
|
|
2260
|
+
since, now_utc, project=None, skip_sync=skip_sync,
|
|
2261
|
+
))
|
|
2262
|
+
if not raw:
|
|
2263
|
+
return _cache_report_empty(
|
|
2264
|
+
today_iso=today_iso, window_days=window_days,
|
|
2265
|
+
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2266
|
+
anomaly_window_days=anomaly_window_days,
|
|
2267
|
+
)
|
|
2268
|
+
built = crk.build_cached_days(
|
|
2269
|
+
_wrap_day_entries(raw), raw,
|
|
2270
|
+
display_tz=display_tz, pricing=pricing,
|
|
2271
|
+
cost_calculator=_calculate_entry_cost,
|
|
2119
2272
|
)
|
|
2120
|
-
|
|
2121
|
-
|
|
2273
|
+
cached_days = dict(built)
|
|
2274
|
+
if use_cache_report_cache:
|
|
2275
|
+
for d, unit in built.items():
|
|
2276
|
+
if d != today_key: # never store the OPEN day as a closed unit
|
|
2277
|
+
_sc.cache_report_day_store(d, unit)
|
|
2278
|
+
# Empty/gap-day sentinel: a genuinely activity-free CLOSED day never
|
|
2279
|
+
# appears in ``built`` (``build_cached_days`` only emits days with
|
|
2280
|
+
# entries), so register an ``is_empty`` sentinel for every needed
|
|
2281
|
+
# closed date the fold produced nothing for. Without this,
|
|
2282
|
+
# ``have_all`` would see that quiet day as a perpetual miss and the
|
|
2283
|
+
# builder would full-window-refetch on EVERY warm tick — permanently
|
|
2284
|
+
# defeating the cache for any user with an off day. Mirrors
|
|
2285
|
+
# ``projects_env_week_put`` registering a computed-empty week as a
|
|
2286
|
+
# hit. ``needed_closed`` already excludes ``today_key``; the sentinel
|
|
2287
|
+
# is skipped in the restitch below, so this is byte-identical.
|
|
2288
|
+
for d in needed_closed:
|
|
2289
|
+
if d not in built:
|
|
2290
|
+
_sc.cache_report_day_store(d, crk.empty_cached_day(d))
|
|
2291
|
+
# Window-rolloff eviction (#275): the reconcile's seq-gated pass only
|
|
2292
|
+
# evicts CHANGED days (>= the change watermark); days that roll off the
|
|
2293
|
+
# trailing edge of the [since, now] window are never pruned there and
|
|
2294
|
+
# would accrete one frozen unit/day on a long-uptime dashboard. Drop
|
|
2295
|
+
# them here on this rare cold/rollover store tick (the warm path never
|
|
2296
|
+
# reaches this branch). ``needed_closed`` is sorted ascending, so its
|
|
2297
|
+
# first element is the window's oldest still-needed closed day; every
|
|
2298
|
+
# needed day (real or ``is_empty`` sentinel) is >= it and survives.
|
|
2299
|
+
if needed_closed:
|
|
2300
|
+
_sc.cache_report_day_evict_before(needed_closed[0])
|
|
2301
|
+
|
|
2302
|
+
if not cached_days:
|
|
2303
|
+
return _cache_report_empty(
|
|
2304
|
+
today_iso=today_iso, window_days=window_days,
|
|
2122
2305
|
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2123
2306
|
anomaly_window_days=anomaly_window_days,
|
|
2124
|
-
today=empty_today,
|
|
2125
|
-
days=(), by_project=(), by_model=(),
|
|
2126
|
-
seven_day_net_usd=0.0,
|
|
2127
|
-
seven_day_anomaly_count=0,
|
|
2128
|
-
fourteen_day_counterfactual_usd=0.0,
|
|
2129
|
-
fourteen_day_efficiency_ratio=0.0,
|
|
2130
|
-
is_empty=True,
|
|
2131
2307
|
)
|
|
2132
2308
|
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
#
|
|
2136
|
-
#
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
day_entries = [
|
|
2142
|
-
_NS(
|
|
2143
|
-
timestamp=e.timestamp,
|
|
2144
|
-
model=e.model,
|
|
2145
|
-
cost_usd=e.cost_usd,
|
|
2146
|
-
usage={
|
|
2147
|
-
"input_tokens": e.input_tokens,
|
|
2148
|
-
"output_tokens": e.output_tokens,
|
|
2149
|
-
"cache_creation_input_tokens": e.cache_creation_tokens,
|
|
2150
|
-
"cache_read_input_tokens": e.cache_read_tokens,
|
|
2151
|
-
},
|
|
2152
|
-
)
|
|
2153
|
-
for e in entries
|
|
2309
|
+
# Reconstruct fresh MUTABLE rows from the frozen cached days (F7 — cached
|
|
2310
|
+
# units are never touched), then run the cross-row classify + summarize
|
|
2311
|
+
# pass over the FULL assembled set every tick (the cached day aggregates
|
|
2312
|
+
# carry no anomaly state).
|
|
2313
|
+
rows = [
|
|
2314
|
+
crk.reconstruct_cache_row(cached_days[d])
|
|
2315
|
+
for d in sorted(cached_days)
|
|
2316
|
+
if not cached_days[d].is_empty # sentinel gap days emit no CacheRow
|
|
2154
2317
|
]
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
day_entries,
|
|
2318
|
+
result = crk.classify_and_summarize(
|
|
2319
|
+
rows,
|
|
2158
2320
|
now_utc=now_utc,
|
|
2159
2321
|
window_days=window_days,
|
|
2160
2322
|
anomaly_threshold_pp=anomaly_threshold_pp,
|
|
2161
2323
|
anomaly_window_days=anomaly_window_days,
|
|
2162
2324
|
display_tz=display_tz,
|
|
2163
|
-
pricing=pricing,
|
|
2164
2325
|
mode="day",
|
|
2165
|
-
cost_calculator=_calculate_entry_cost,
|
|
2166
2326
|
)
|
|
2167
2327
|
|
|
2168
2328
|
# Pick out today's row (if any) and the baseline-daily-row count for
|
|
@@ -2299,17 +2459,22 @@ def build_cache_report_snapshot(
|
|
|
2299
2459
|
# cache entry and its corresponding day row always agree on which
|
|
2300
2460
|
# bucket they belong to.
|
|
2301
2461
|
kept_dates = frozenset(r.date for r in days if r.date)
|
|
2302
|
-
bucket_tz = display_tz if display_tz is not None else dt.timezone.utc
|
|
2303
|
-
entries_in_window = [
|
|
2304
|
-
e for e in entries
|
|
2305
|
-
if e.timestamp.astimezone(bucket_tz).strftime("%Y-%m-%d") in kept_dates
|
|
2306
|
-
]
|
|
2307
2462
|
rows_in_window = [r for r in result.rows if r.date in kept_dates]
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2463
|
+
# by_project via the #272 §4 two-level ``stable_sum`` fold
|
|
2464
|
+
# (grouping-invariant). The per-(day, project) partials are carried on each
|
|
2465
|
+
# ``CachedCacheReportDay`` (§5) — served from cache for closed days and
|
|
2466
|
+
# freshly folded for today — so restitching them here memoizes the fold
|
|
2467
|
+
# byte-identically. Their day keys use ``_resolve_bucket_tz(display_tz)``
|
|
2468
|
+
# (matching ``CacheRow.date`` / ``kept_dates``), so combining over
|
|
2469
|
+
# ``kept_dates`` is exactly the old full-window slice, modulo the one-time
|
|
2470
|
+
# ULP fold move. Every ``kept_date`` is in ``cached_days`` (it came from a
|
|
2471
|
+
# day row); the ``if d in cached_days`` guard is defensive.
|
|
2472
|
+
by_project_rows = crk.combine_day_project_partials(
|
|
2473
|
+
{
|
|
2474
|
+
d: dict(cached_days[d].project_partials)
|
|
2475
|
+
for d in kept_dates
|
|
2476
|
+
if d in cached_days and not cached_days[d].is_empty
|
|
2477
|
+
}
|
|
2313
2478
|
)
|
|
2314
2479
|
by_model_rows = crk._aggregate_cache_breakdown_from_rows(
|
|
2315
2480
|
rows_in_window,
|
|
@@ -2629,6 +2794,33 @@ def _group_a_monthly_buckets(now_utc, *, n, range_start, display_tz):
|
|
|
2629
2794
|
_start, nxt = _month_bounds(label)
|
|
2630
2795
|
return nxt + dt.timedelta(days=2)
|
|
2631
2796
|
|
|
2797
|
+
# #271: current-bucket accumulator inputs. `_membership` IS
|
|
2798
|
+
# `_aggregate_monthly`'s own key (display-tz `%Y-%m`);
|
|
2799
|
+
# `_all_fetch`/`_delta_fetch` reuse `_fetch`'s ±2-day-padded month
|
|
2800
|
+
# window (clamped to now_utc) via the `(id, UsageEntry)` delta sibling.
|
|
2801
|
+
def _membership(e):
|
|
2802
|
+
local = (
|
|
2803
|
+
e.timestamp.astimezone(display_tz) if display_tz is not None
|
|
2804
|
+
# internal fallback: host-local intentional
|
|
2805
|
+
else e.timestamp.astimezone()
|
|
2806
|
+
)
|
|
2807
|
+
return local.strftime("%Y-%m") == current_label
|
|
2808
|
+
|
|
2809
|
+
def _current_window(label):
|
|
2810
|
+
start, nxt = _month_bounds(label)
|
|
2811
|
+
lo = max(start - dt.timedelta(days=2), range_start)
|
|
2812
|
+
hi = min(nxt + dt.timedelta(days=2), now_utc)
|
|
2813
|
+
return lo, hi
|
|
2814
|
+
|
|
2815
|
+
def _all_fetch(label):
|
|
2816
|
+
lo, hi = _current_window(label)
|
|
2817
|
+
return [] if hi <= lo else iter_entries_with_id(cache_conn, lo, hi)
|
|
2818
|
+
|
|
2819
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
2820
|
+
lo, hi = _current_window(label)
|
|
2821
|
+
return [] if hi <= lo else iter_entries_with_id(
|
|
2822
|
+
cache_conn, lo, hi, after_seq=after_seq, after_ts=after_ts)
|
|
2823
|
+
|
|
2632
2824
|
buckets = build_cached_group_a(
|
|
2633
2825
|
"monthly",
|
|
2634
2826
|
cache_conn=cache_conn,
|
|
@@ -2638,6 +2830,11 @@ def _group_a_monthly_buckets(now_utc, *, n, range_start, display_tz):
|
|
|
2638
2830
|
fetch_bucket_entries=_fetch,
|
|
2639
2831
|
aggregate_one=_agg_one,
|
|
2640
2832
|
extra_signature=("monthly", tz_sig),
|
|
2833
|
+
use_current_accumulator=True,
|
|
2834
|
+
now_utc=now_utc,
|
|
2835
|
+
current_all_fetch=_all_fetch,
|
|
2836
|
+
current_delta_fetch=_delta_fetch,
|
|
2837
|
+
membership_of=_membership,
|
|
2641
2838
|
)
|
|
2642
2839
|
return tuple(buckets)
|
|
2643
2840
|
finally:
|
|
@@ -2787,6 +2984,52 @@ def _group_a_weekly_buckets(stats_conn, now_utc, *, weeks):
|
|
|
2787
2984
|
except ValueError:
|
|
2788
2985
|
return None
|
|
2789
2986
|
|
|
2987
|
+
# #271: current-bucket accumulator inputs. Membership REUSES
|
|
2988
|
+
# _aggregate_weekly's exact bisect + first-match-wins assignment
|
|
2989
|
+
# (bin/_lib_aggregators.py::_week_key_or_none) — the parsed-bounds list
|
|
2990
|
+
# is built identically (same order, same parse) so an entry in an
|
|
2991
|
+
# overlap region (reset-day drift) keys to the SAME week the full pass
|
|
2992
|
+
# assigns it. _all_fetch/_delta_fetch reuse _fetch's
|
|
2993
|
+
# [week.start, min(week.end, now_utc)] window via the (id, UsageEntry)
|
|
2994
|
+
# delta sibling.
|
|
2995
|
+
_parsed_bounds = [
|
|
2996
|
+
(parse_iso_datetime(w.start_ts, "week.start_ts"),
|
|
2997
|
+
parse_iso_datetime(w.end_ts, "week.end_ts"),
|
|
2998
|
+
w.start_date.isoformat())
|
|
2999
|
+
for w in weeks
|
|
3000
|
+
]
|
|
3001
|
+
_bound_starts = [b[0] for b in _parsed_bounds]
|
|
3002
|
+
|
|
3003
|
+
def _week_key(e):
|
|
3004
|
+
ts = e.timestamp
|
|
3005
|
+
idx = bisect.bisect_right(_bound_starts, ts) - 1
|
|
3006
|
+
if idx < 0:
|
|
3007
|
+
return None
|
|
3008
|
+
while idx > 0 and (
|
|
3009
|
+
_parsed_bounds[idx - 1][0] <= ts < _parsed_bounds[idx - 1][1]
|
|
3010
|
+
):
|
|
3011
|
+
idx -= 1
|
|
3012
|
+
s_dt, e_dt, key = _parsed_bounds[idx]
|
|
3013
|
+
return key if s_dt <= ts < e_dt else None
|
|
3014
|
+
|
|
3015
|
+
def _membership(e):
|
|
3016
|
+
return _week_key(e) == current_label
|
|
3017
|
+
|
|
3018
|
+
def _current_window(label):
|
|
3019
|
+
w = sw_by_label[label]
|
|
3020
|
+
s = parse_iso_datetime(w.start_ts, "week.start_ts")
|
|
3021
|
+
e = parse_iso_datetime(w.end_ts, "week.end_ts")
|
|
3022
|
+
return s, min(e, now_utc)
|
|
3023
|
+
|
|
3024
|
+
def _all_fetch(label):
|
|
3025
|
+
s, hi = _current_window(label)
|
|
3026
|
+
return [] if hi <= s else iter_entries_with_id(cache_conn, s, hi)
|
|
3027
|
+
|
|
3028
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
3029
|
+
s, hi = _current_window(label)
|
|
3030
|
+
return [] if hi <= s else iter_entries_with_id(
|
|
3031
|
+
cache_conn, s, hi, after_seq=after_seq, after_ts=after_ts)
|
|
3032
|
+
|
|
2790
3033
|
buckets = build_cached_group_a(
|
|
2791
3034
|
"weekly",
|
|
2792
3035
|
cache_conn=cache_conn,
|
|
@@ -2796,6 +3039,11 @@ def _group_a_weekly_buckets(stats_conn, now_utc, *, weeks):
|
|
|
2796
3039
|
fetch_bucket_entries=_fetch,
|
|
2797
3040
|
aggregate_one=_agg_one,
|
|
2798
3041
|
extra_signature=extra_signature,
|
|
3042
|
+
use_current_accumulator=True,
|
|
3043
|
+
now_utc=now_utc,
|
|
3044
|
+
current_all_fetch=_all_fetch,
|
|
3045
|
+
current_delta_fetch=_delta_fetch,
|
|
3046
|
+
membership_of=_membership,
|
|
2799
3047
|
)
|
|
2800
3048
|
return tuple(buckets)
|
|
2801
3049
|
finally:
|
|
@@ -2806,7 +3054,8 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
2806
3054
|
now_utc: "dt.datetime",
|
|
2807
3055
|
*, n: int = 12,
|
|
2808
3056
|
skip_sync: bool = False,
|
|
2809
|
-
use_group_a_cache: bool = False
|
|
3057
|
+
use_group_a_cache: bool = False,
|
|
3058
|
+
use_bugk_segment_cache: bool = False) -> "list[WeeklyPeriodRow]":
|
|
2810
3059
|
"""Latest n subscription weeks as WeeklyPeriodRow, newest-first.
|
|
2811
3060
|
|
|
2812
3061
|
Thin builder-using prelude + Bug-K pre-credit synthesis on top of
|
|
@@ -3019,23 +3268,59 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
3019
3268
|
# or the fallback's wide fetch above did) rather than filtering a
|
|
3020
3269
|
# wide `entries` list, so Bug-K works identically on the cached
|
|
3021
3270
|
# path (where no wide entry list exists) and the fallback path.
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3271
|
+
#
|
|
3272
|
+
# #271 §18: the [original_start, effective) window is a CLOSED past
|
|
3273
|
+
# interval (effective is a historical credit moment), so this folded
|
|
3274
|
+
# aggregate is IMMUTABLE — cache it byte-identically (the same
|
|
3275
|
+
# "re-aggregate immutable history every tick" pattern the #269
|
|
3276
|
+
# weekref cost cache fixed). `_compute_pre_segment` is the exact
|
|
3277
|
+
# from-scratch fetch+fold closure; on the sync thread
|
|
3278
|
+
# (`use_bugk_segment_cache=True`) it routes through
|
|
3279
|
+
# `cached_bugk_segment` (get-or-compute over the canonical window
|
|
3280
|
+
# key), on EVERY other caller (CLI / share / tests / reconcile-
|
|
3281
|
+
# failed) it computes directly — byte-unchanged. The `used_pct`
|
|
3282
|
+
# snapshot query + WeeklyPeriodRow build BELOW always rerun fresh
|
|
3283
|
+
# from the segment (do NOT cache the row — Codex-BK-5).
|
|
3284
|
+
def _compute_pre_segment(_orig=original_start_dt, _eff=eff_dt):
|
|
3285
|
+
pi = po = pcc = pcr = 0
|
|
3286
|
+
pcost = 0.0
|
|
3287
|
+
pmodels: dict[str, float] = {}
|
|
3288
|
+
pcount = 0
|
|
3289
|
+
for e in get_entries(_orig, _eff, skip_sync=True):
|
|
3290
|
+
if _orig <= e.timestamp < _eff:
|
|
3291
|
+
usage = e.usage
|
|
3292
|
+
pi += usage.get("input_tokens", 0)
|
|
3293
|
+
po += usage.get("output_tokens", 0)
|
|
3294
|
+
pcc += usage.get("cache_creation_input_tokens", 0)
|
|
3295
|
+
pcr += usage.get("cache_read_input_tokens", 0)
|
|
3296
|
+
c = _calc(
|
|
3297
|
+
e.model, usage, mode="auto", cost_usd=e.cost_usd,
|
|
3298
|
+
)
|
|
3299
|
+
pcost += c
|
|
3300
|
+
pmodels[e.model] = pmodels.get(e.model, 0.0) + c
|
|
3301
|
+
pcount += 1
|
|
3302
|
+
# Codex-BK-4: freeze `models` as a tuple of (model, cost) in
|
|
3303
|
+
# first-seen (dict-insertion) order so the row's stable cost-desc
|
|
3304
|
+
# sort tie-order can't be mutated after caching.
|
|
3305
|
+
return BugKSegment(
|
|
3306
|
+
input=pi, output=po, cache_create=pcc, cache_read=pcr,
|
|
3307
|
+
cost=pcost, models=tuple(pmodels.items()), entry_count=pcount,
|
|
3308
|
+
)
|
|
3309
|
+
|
|
3310
|
+
if use_bugk_segment_cache:
|
|
3311
|
+
seg = cached_bugk_segment(
|
|
3312
|
+
key=_bugk_key(original_start_dt, eff_dt),
|
|
3313
|
+
compute=_compute_pre_segment,
|
|
3314
|
+
)
|
|
3315
|
+
else:
|
|
3316
|
+
seg = _compute_pre_segment()
|
|
3317
|
+
pre_input = seg.input
|
|
3318
|
+
pre_output = seg.output
|
|
3319
|
+
pre_cc = seg.cache_create
|
|
3320
|
+
pre_cr = seg.cache_read
|
|
3321
|
+
pre_cost = seg.cost
|
|
3322
|
+
pre_models = seg.models # ((model, cost), ...) in first-seen order
|
|
3323
|
+
pre_entry_count = seg.entry_count
|
|
3039
3324
|
if pre_entry_count == 0 and pre_cost <= 0:
|
|
3040
3325
|
# No measurable pre-credit activity — skip insertion.
|
|
3041
3326
|
continue
|
|
@@ -3058,9 +3343,12 @@ def _dashboard_build_weekly_periods(conn: "sqlite3.Connection",
|
|
|
3058
3343
|
)
|
|
3059
3344
|
|
|
3060
3345
|
pre_total = pre_input + pre_output + pre_cc + pre_cr
|
|
3346
|
+
# `pre_models` is a first-seen-order tuple of (model, cost); the
|
|
3347
|
+
# stable cost-desc sort preserves that tie-order byte-for-byte,
|
|
3348
|
+
# identical to the pre-#271 `sorted(dict.items(), ...)`.
|
|
3061
3349
|
pre_model_breakdowns = [
|
|
3062
3350
|
{"modelName": m, "cost": c}
|
|
3063
|
-
for m, c in sorted(pre_models
|
|
3351
|
+
for m, c in sorted(pre_models, key=lambda kv: -kv[1])
|
|
3064
3352
|
]
|
|
3065
3353
|
pre_label = original_start_dt.strftime("%m-%d")
|
|
3066
3354
|
pre_row = WeeklyPeriodRow(
|
|
@@ -3362,6 +3650,36 @@ def _group_a_daily_buckets(now_utc, *, n, display_tz):
|
|
|
3362
3650
|
+ dt.timedelta(days=2)
|
|
3363
3651
|
)
|
|
3364
3652
|
|
|
3653
|
+
# #271: current-bucket accumulator inputs. `_membership` IS
|
|
3654
|
+
# `_aggregate_daily`'s own key (display-tz `%Y-%m-%d`), so the id-bounded
|
|
3655
|
+
# fetch keeps exactly the entries the full pass assigns to the current
|
|
3656
|
+
# day, byte-identically. `_all_fetch`/`_delta_fetch` reuse `_fetch`'s
|
|
3657
|
+
# ±1-day-padded window (clamped to now_utc) via the `(id, UsageEntry)`
|
|
3658
|
+
# delta sibling.
|
|
3659
|
+
def _membership(e):
|
|
3660
|
+
local = (
|
|
3661
|
+
e.timestamp.astimezone(display_tz) if display_tz is not None
|
|
3662
|
+
# internal fallback: host-local intentional
|
|
3663
|
+
else e.timestamp.astimezone()
|
|
3664
|
+
)
|
|
3665
|
+
return local.strftime("%Y-%m-%d") == current_label
|
|
3666
|
+
|
|
3667
|
+
def _current_window(label):
|
|
3668
|
+
d = dt.date.fromisoformat(label)
|
|
3669
|
+
base = dt.datetime(d.year, d.month, d.day, tzinfo=dt.timezone.utc)
|
|
3670
|
+
lo = max(base - dt.timedelta(days=1), range_start)
|
|
3671
|
+
hi = min(base + dt.timedelta(days=2), now_utc)
|
|
3672
|
+
return lo, hi
|
|
3673
|
+
|
|
3674
|
+
def _all_fetch(label):
|
|
3675
|
+
lo, hi = _current_window(label)
|
|
3676
|
+
return [] if hi <= lo else iter_entries_with_id(cache_conn, lo, hi)
|
|
3677
|
+
|
|
3678
|
+
def _delta_fetch(label, after_seq, after_ts):
|
|
3679
|
+
lo, hi = _current_window(label)
|
|
3680
|
+
return [] if hi <= lo else iter_entries_with_id(
|
|
3681
|
+
cache_conn, lo, hi, after_seq=after_seq, after_ts=after_ts)
|
|
3682
|
+
|
|
3365
3683
|
buckets = build_cached_group_a(
|
|
3366
3684
|
"daily",
|
|
3367
3685
|
cache_conn=cache_conn,
|
|
@@ -3371,6 +3689,11 @@ def _group_a_daily_buckets(now_utc, *, n, display_tz):
|
|
|
3371
3689
|
fetch_bucket_entries=_fetch,
|
|
3372
3690
|
aggregate_one=_agg_one,
|
|
3373
3691
|
extra_signature=("daily", tz_sig),
|
|
3692
|
+
use_current_accumulator=True,
|
|
3693
|
+
now_utc=now_utc,
|
|
3694
|
+
current_all_fetch=_all_fetch,
|
|
3695
|
+
current_delta_fetch=_delta_fetch,
|
|
3696
|
+
membership_of=_membership,
|
|
3374
3697
|
)
|
|
3375
3698
|
return tuple(buckets)
|
|
3376
3699
|
finally:
|
|
@@ -3545,13 +3868,41 @@ def _projects_week_label(week_start: "dt.datetime") -> str:
|
|
|
3545
3868
|
def _projects_iter_session_entries(conn: "sqlite3.Connection",
|
|
3546
3869
|
*,
|
|
3547
3870
|
since: "dt.datetime",
|
|
3548
|
-
until: "dt.datetime"
|
|
3871
|
+
until: "dt.datetime",
|
|
3872
|
+
after_seq: "int | None" = None):
|
|
3549
3873
|
"""Read ``session_entries`` joined with ``session_files`` over
|
|
3550
3874
|
[since, until]. Yields rows directly off the passed conn — no
|
|
3551
3875
|
cache.db monkeypatch, no production ``get_claude_session_entries``
|
|
3552
3876
|
pipeline. The fixture DBs co-locate both schemas in one file; the
|
|
3553
3877
|
production wiring opens both DBs and ATTACHes cache.db as a schema
|
|
3554
3878
|
on the stats conn (see ``_run_dashboard_sync_tick``).
|
|
3879
|
+
|
|
3880
|
+
``after_seq`` (#270 §8, re-key of Codex-P2b — #271 §20): when set, the
|
|
3881
|
+
current-week accumulator's warm delta seeks by the **mutation_seq range**
|
|
3882
|
+
(``WHERE e.mutation_seq > ?``) with the ``[since, until]`` timestamp bounds
|
|
3883
|
+
applied as a residual filter. Two hints keep the planner on the
|
|
3884
|
+
``idx_entries_mutation_seq`` seek (``SEARCH e USING INDEX
|
|
3885
|
+
idx_entries_mutation_seq (mutation_seq>?)``) instead of a full ``SCAN`` /
|
|
3886
|
+
``idx_entries_timestamp`` window scan (which would re-read the whole ~12K-entry
|
|
3887
|
+
current week and let the ~63ms floor creep back): (1) the unary-plus no-op
|
|
3888
|
+
``+e.timestamp_utc`` deprioritizes ``idx_entries_timestamp`` (a TRUE no-op —
|
|
3889
|
+
it preserves the TEXT value AND the string comparison, unlike ``+ 0``); and
|
|
3890
|
+
(2) ``ORDER BY e.mutation_seq`` matches the index's leading column so the
|
|
3891
|
+
planner satisfies the order from the seek itself. ``INDEXED BY`` is NOT usable
|
|
3892
|
+
here — production runs this against a TEMP VIEW (``session_entries`` over the
|
|
3893
|
+
ATTACHed ``cache_db.session_entries``), and a view has no indexes; the hint
|
|
3894
|
+
pair drives the same seek on both the view and a base-table fixture. Unlike
|
|
3895
|
+
the old ``e.id > ?`` leg (a free INTEGER PRIMARY KEY rowid range), a bare
|
|
3896
|
+
``e.mutation_seq > ?`` lets the planner fall back to a scan on a small table,
|
|
3897
|
+
so the hints are load-bearing. Re-keying from ``e.id`` to ``e.mutation_seq``
|
|
3898
|
+
catches an id-stable in-place finalization (it advances the seq while
|
|
3899
|
+
``MAX(id)`` stays flat, so the pre-#270 ``e.id > ?`` leg missed it); on a
|
|
3900
|
+
pure-insert tick ``{mutation_seq > ?}`` == ``{id > ?}`` (seq monotone with
|
|
3901
|
+
id), so the delta row SET is byte-identical. The caller (``_fetch_delta_rows``)
|
|
3902
|
+
re-sorts the small result by ``(timestamp_utc, id)`` to reproduce the full
|
|
3903
|
+
pass's fold order, so the ``mutation_seq``-ASC SQL order is irrelevant
|
|
3904
|
+
downstream. An ``EXPLAIN QUERY PLAN`` regression asserts the mutation_seq
|
|
3905
|
+
index seek (``tests/test_projects_envelope.py``).
|
|
3555
3906
|
"""
|
|
3556
3907
|
since_iso = since.astimezone(dt.timezone.utc).strftime(
|
|
3557
3908
|
"%Y-%m-%dT%H:%M:%SZ"
|
|
@@ -3559,17 +3910,30 @@ def _projects_iter_session_entries(conn: "sqlite3.Connection",
|
|
|
3559
3910
|
until_iso = until.astimezone(dt.timezone.utc).strftime(
|
|
3560
3911
|
"%Y-%m-%dT%H:%M:%SZ"
|
|
3561
3912
|
)
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3568
|
-
|
|
3569
|
-
|
|
3570
|
-
|
|
3571
|
-
|
|
3572
|
-
|
|
3913
|
+
if after_seq is not None:
|
|
3914
|
+
cur = conn.execute(
|
|
3915
|
+
"SELECT e.id, e.timestamp_utc, e.model, e.input_tokens, "
|
|
3916
|
+
" e.output_tokens, e.cache_create_tokens, e.cache_read_tokens, "
|
|
3917
|
+
" e.cost_usd_raw, e.source_path, "
|
|
3918
|
+
" sf.session_id, sf.project_path "
|
|
3919
|
+
"FROM session_entries e "
|
|
3920
|
+
"LEFT JOIN session_files sf ON sf.path = e.source_path "
|
|
3921
|
+
"WHERE e.mutation_seq > ? AND +e.timestamp_utc >= ? AND +e.timestamp_utc <= ? "
|
|
3922
|
+
"ORDER BY e.mutation_seq ASC",
|
|
3923
|
+
(after_seq, since_iso, until_iso),
|
|
3924
|
+
)
|
|
3925
|
+
else:
|
|
3926
|
+
cur = conn.execute(
|
|
3927
|
+
"SELECT e.id, e.timestamp_utc, e.model, e.input_tokens, "
|
|
3928
|
+
" e.output_tokens, e.cache_create_tokens, e.cache_read_tokens, "
|
|
3929
|
+
" e.cost_usd_raw, e.source_path, "
|
|
3930
|
+
" sf.session_id, sf.project_path "
|
|
3931
|
+
"FROM session_entries e "
|
|
3932
|
+
"LEFT JOIN session_files sf ON sf.path = e.source_path "
|
|
3933
|
+
"WHERE e.timestamp_utc >= ? AND e.timestamp_utc <= ? "
|
|
3934
|
+
"ORDER BY e.timestamp_utc ASC, e.id ASC",
|
|
3935
|
+
(since_iso, until_iso),
|
|
3936
|
+
)
|
|
3573
3937
|
for row in cur:
|
|
3574
3938
|
yield row
|
|
3575
3939
|
|
|
@@ -3600,77 +3964,121 @@ class _ProjWeekBucket(NamedTuple):
|
|
|
3600
3964
|
first_key: "Any"
|
|
3601
3965
|
|
|
3602
3966
|
|
|
3603
|
-
def
|
|
3967
|
+
def _fold_projects_entry(
|
|
3968
|
+
mut: dict,
|
|
3969
|
+
row: tuple,
|
|
3970
|
+
*,
|
|
3971
|
+
resolver_cache: dict,
|
|
3972
|
+
week_start: "dt.datetime",
|
|
3973
|
+
) -> "float | None":
|
|
3974
|
+
"""Fold ONE ``_projects_iter_session_entries`` row onto ``mut`` (the shared
|
|
3975
|
+
per-row body, #271 §20 Codex-P1a).
|
|
3976
|
+
|
|
3977
|
+
Used by BOTH the full-window cold fold (``_aggregate_projects_week_raw``)
|
|
3978
|
+
and the current-week accumulator's warm incremental append, so their
|
|
3979
|
+
per-bucket cost sums, session sets, and first/last-seen captures are
|
|
3980
|
+
byte-identical BY CONSTRUCTION. Returns the entry cost, or ``None`` when the
|
|
3981
|
+
row is filtered out (``<synthetic>`` model, or its Monday-anchored week ≠
|
|
3982
|
+
``week_start``) — the caller then skips ``week_total`` / ``tail`` advance.
|
|
3983
|
+
|
|
3984
|
+
``mut[bp]`` is the running mutable dict ``{"cost_usd": float,
|
|
3985
|
+
"sessions": set, "first_seen": dt, "last_seen": dt, "first_order": ts_iso,
|
|
3986
|
+
"first_id": int, "first_key": ProjectKey}``. The first row seen for a
|
|
3987
|
+
``bucket_path`` captures ``first_order`` / ``first_id`` / ``first_key`` —
|
|
3988
|
+
order-safe for the warm append because every delta row sorts strictly after
|
|
3989
|
+
``tail`` (#271 §20), so a delta row is never the first-seen of a bucket that
|
|
3990
|
+
already exists in ``mut``.
|
|
3991
|
+
"""
|
|
3992
|
+
c = _cctally()
|
|
3993
|
+
(entry_id, ts_iso, model, input_tok, output_tok,
|
|
3994
|
+
cache_create, cache_read, cost_raw, source_path,
|
|
3995
|
+
session_id, project_path) = row
|
|
3996
|
+
if model == "<synthetic>":
|
|
3997
|
+
return None
|
|
3998
|
+
ts = parse_iso_datetime(ts_iso, "session_entries.timestamp_utc")
|
|
3999
|
+
if _projects_week_start_monday_utc(ts) != week_start:
|
|
4000
|
+
return None
|
|
4001
|
+
entry_cost = _calculate_entry_cost(
|
|
4002
|
+
model,
|
|
4003
|
+
{
|
|
4004
|
+
"input_tokens": input_tok or 0,
|
|
4005
|
+
"output_tokens": output_tok or 0,
|
|
4006
|
+
"cache_creation_input_tokens": cache_create or 0,
|
|
4007
|
+
"cache_read_input_tokens": cache_read or 0,
|
|
4008
|
+
},
|
|
4009
|
+
mode="auto",
|
|
4010
|
+
cost_usd=cost_raw,
|
|
4011
|
+
)
|
|
4012
|
+
pkey = c._resolve_project_key(project_path, "git-root", resolver_cache)
|
|
4013
|
+
bp = pkey.bucket_path
|
|
4014
|
+
a = mut.get(bp)
|
|
4015
|
+
if a is None:
|
|
4016
|
+
a = {
|
|
4017
|
+
"cost_usd": 0.0,
|
|
4018
|
+
"sessions": set(),
|
|
4019
|
+
"first_seen": ts,
|
|
4020
|
+
"last_seen": ts,
|
|
4021
|
+
"first_order": ts_iso,
|
|
4022
|
+
"first_id": entry_id,
|
|
4023
|
+
"first_key": pkey,
|
|
4024
|
+
}
|
|
4025
|
+
mut[bp] = a
|
|
4026
|
+
a["cost_usd"] += entry_cost
|
|
4027
|
+
if session_id:
|
|
4028
|
+
a["sessions"].add(session_id)
|
|
4029
|
+
elif source_path:
|
|
4030
|
+
a["sessions"].add(source_path)
|
|
4031
|
+
if ts < a["first_seen"]:
|
|
4032
|
+
a["first_seen"] = ts
|
|
4033
|
+
if ts > a["last_seen"]:
|
|
4034
|
+
a["last_seen"] = ts
|
|
4035
|
+
return entry_cost
|
|
4036
|
+
|
|
4037
|
+
|
|
4038
|
+
def _aggregate_projects_week_raw(
|
|
3604
4039
|
conn: "sqlite3.Connection",
|
|
3605
4040
|
*,
|
|
3606
4041
|
week_start: "dt.datetime",
|
|
3607
4042
|
week_end: "dt.datetime",
|
|
3608
4043
|
resolver_cache: dict,
|
|
3609
|
-
) -> "tuple[dict
|
|
3610
|
-
"""
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
3620
|
-
|
|
3621
|
-
|
|
4044
|
+
) -> "tuple[dict, float, tuple | None]":
|
|
4045
|
+
"""Full-window fold for one Monday-anchored subscription week, returning the
|
|
4046
|
+
RAW mutable ``mut`` (sessions kept as SETS, so the current-week accumulator
|
|
4047
|
+
can dedup across ticks), the entry-order ``week_total`` float, and the
|
|
4048
|
+
``tail`` ``(ts_iso, id)`` of the last folded CURRENT-WEEK entry (``None``
|
|
4049
|
+
when no row folded).
|
|
4050
|
+
|
|
4051
|
+
Shared by the finalizing ``_aggregate_projects_week`` wrapper (closed-week
|
|
4052
|
+
cache path) and the current-week accumulator's cold seed (#271 §20
|
|
4053
|
+
Codex-P1a — the finalized public shape discards the sessions SET into a
|
|
4054
|
+
count, so cold seeding needs the raw ``mut``). Byte-identical to the
|
|
4055
|
+
original single-loop aggregate: same ``timestamp_utc ASC, id ASC`` order,
|
|
4056
|
+
same per-row ``_fold_projects_entry`` arithmetic, same entry-order
|
|
4057
|
+
``week_total`` left-fold.
|
|
3622
4058
|
"""
|
|
3623
|
-
c = _cctally()
|
|
3624
|
-
_resolve_project_key = c._resolve_project_key
|
|
3625
|
-
week_total = 0.0
|
|
3626
4059
|
mut: "dict[str, dict]" = {}
|
|
4060
|
+
week_total = 0.0
|
|
4061
|
+
tail: "tuple | None" = None
|
|
3627
4062
|
for row in _projects_iter_session_entries(
|
|
3628
4063
|
conn, since=week_start, until=week_end,
|
|
3629
4064
|
):
|
|
3630
|
-
|
|
3631
|
-
|
|
3632
|
-
session_id, project_path) = row
|
|
3633
|
-
if model == "<synthetic>":
|
|
3634
|
-
continue
|
|
3635
|
-
ts = parse_iso_datetime(ts_iso, "session_entries.timestamp_utc")
|
|
3636
|
-
if _projects_week_start_monday_utc(ts) != week_start:
|
|
3637
|
-
continue
|
|
3638
|
-
entry_cost = _calculate_entry_cost(
|
|
3639
|
-
model,
|
|
3640
|
-
{
|
|
3641
|
-
"input_tokens": input_tok or 0,
|
|
3642
|
-
"output_tokens": output_tok or 0,
|
|
3643
|
-
"cache_creation_input_tokens": cache_create or 0,
|
|
3644
|
-
"cache_read_input_tokens": cache_read or 0,
|
|
3645
|
-
},
|
|
3646
|
-
mode="auto",
|
|
3647
|
-
cost_usd=cost_raw,
|
|
4065
|
+
entry_cost = _fold_projects_entry(
|
|
4066
|
+
mut, row, resolver_cache=resolver_cache, week_start=week_start,
|
|
3648
4067
|
)
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
a = mut.get(bp)
|
|
3652
|
-
if a is None:
|
|
3653
|
-
a = {
|
|
3654
|
-
"cost_usd": 0.0,
|
|
3655
|
-
"sessions": set(),
|
|
3656
|
-
"first_seen": ts,
|
|
3657
|
-
"last_seen": ts,
|
|
3658
|
-
"first_order": ts_iso,
|
|
3659
|
-
"first_id": entry_id,
|
|
3660
|
-
"first_key": pkey,
|
|
3661
|
-
}
|
|
3662
|
-
mut[bp] = a
|
|
3663
|
-
a["cost_usd"] += entry_cost
|
|
3664
|
-
if session_id:
|
|
3665
|
-
a["sessions"].add(session_id)
|
|
3666
|
-
elif source_path:
|
|
3667
|
-
a["sessions"].add(source_path)
|
|
3668
|
-
if ts < a["first_seen"]:
|
|
3669
|
-
a["first_seen"] = ts
|
|
3670
|
-
if ts > a["last_seen"]:
|
|
3671
|
-
a["last_seen"] = ts
|
|
4068
|
+
if entry_cost is None:
|
|
4069
|
+
continue
|
|
3672
4070
|
week_total += entry_cost
|
|
3673
|
-
|
|
4071
|
+
tail = (row[1], row[0]) # (ts_iso, id)
|
|
4072
|
+
return mut, week_total, tail
|
|
4073
|
+
|
|
4074
|
+
|
|
4075
|
+
def _finalize_projects_mut(mut: dict) -> "dict[str, _ProjWeekBucket]":
|
|
4076
|
+
"""Finalize a raw ``mut`` (sessions SETS) into ``{bucket_path:
|
|
4077
|
+
_ProjWeekBucket}`` — ``sessions_count = len(sessions)``, the rest copied
|
|
4078
|
+
through. The public shape both the closed-week cache and the current-week
|
|
4079
|
+
accumulator return (#271 §20).
|
|
4080
|
+
"""
|
|
4081
|
+
return {
|
|
3674
4082
|
bp: _ProjWeekBucket(
|
|
3675
4083
|
cost_usd=a["cost_usd"],
|
|
3676
4084
|
sessions_count=len(a["sessions"]),
|
|
@@ -3682,7 +4090,37 @@ def _aggregate_projects_week(
|
|
|
3682
4090
|
)
|
|
3683
4091
|
for bp, a in mut.items()
|
|
3684
4092
|
}
|
|
3685
|
-
|
|
4093
|
+
|
|
4094
|
+
|
|
4095
|
+
def _aggregate_projects_week(
|
|
4096
|
+
conn: "sqlite3.Connection",
|
|
4097
|
+
*,
|
|
4098
|
+
week_start: "dt.datetime",
|
|
4099
|
+
week_end: "dt.datetime",
|
|
4100
|
+
resolver_cache: dict,
|
|
4101
|
+
) -> "tuple[dict[str, _ProjWeekBucket], float]":
|
|
4102
|
+
"""Aggregate one Monday-anchored subscription week's entry slice.
|
|
4103
|
+
|
|
4104
|
+
Returns ``({bucket_path: _ProjWeekBucket}, week_total_cost)``. Byte-identical
|
|
4105
|
+
to the contribution the full-window walk in ``_build_projects_envelope``
|
|
4106
|
+
makes for this week: the per-week query returns entries in the same
|
|
4107
|
+
``timestamp_utc ASC, id ASC`` order, filtered to those whose
|
|
4108
|
+
Monday-anchored week equals ``week_start``, so the per-bucket cost sum, the
|
|
4109
|
+
entry-order ``week_total`` sum, the session set, and the first/last-seen +
|
|
4110
|
+
SQL-first-entry captures all reproduce the full pass. ``week_end`` is the
|
|
4111
|
+
inclusive query bound (``<= ?``); the exact next-Monday boundary entry is
|
|
4112
|
+
fetched but filtered out (it belongs to the next week) — the next week's
|
|
4113
|
+
slice keeps it, so no entry is lost or double-counted.
|
|
4114
|
+
|
|
4115
|
+
Thin wrapper over ``_aggregate_projects_week_raw`` + ``_finalize_projects_mut``
|
|
4116
|
+
(#271 §20 Codex-P1a); the public shape is unchanged for the closed-week
|
|
4117
|
+
cache path.
|
|
4118
|
+
"""
|
|
4119
|
+
mut, week_total, _tail = _aggregate_projects_week_raw(
|
|
4120
|
+
conn, week_start=week_start, week_end=week_end,
|
|
4121
|
+
resolver_cache=resolver_cache,
|
|
4122
|
+
)
|
|
4123
|
+
return _finalize_projects_mut(mut), week_total
|
|
3686
4124
|
|
|
3687
4125
|
|
|
3688
4126
|
def _assemble_projects_via_cache(
|
|
@@ -3691,6 +4129,8 @@ def _assemble_projects_via_cache(
|
|
|
3691
4129
|
weeks_full: "list[dt.datetime]",
|
|
3692
4130
|
cw_start: "dt.datetime",
|
|
3693
4131
|
cw_end: "dt.datetime",
|
|
4132
|
+
cur_max_id: int,
|
|
4133
|
+
cur_max_seq: int,
|
|
3694
4134
|
) -> "tuple[dict, dict, dict]":
|
|
3695
4135
|
"""Flag-ON assembly (#269 §14 Win 2): recompute only the CURRENT week each
|
|
3696
4136
|
warm tick; serve CLOSED weeks from the per-(project, week) cache on a hit and
|
|
@@ -3698,6 +4138,16 @@ def _assemble_projects_via_cache(
|
|
|
3698
4138
|
Codex-M4 P2). Returns ``(buckets, total_cost_by_week, key_by_bucket)`` in the
|
|
3699
4139
|
exact shape the from-scratch walk produces, so the downstream disambiguation
|
|
3700
4140
|
/ attribution / trend assembly runs unchanged and byte-identically.
|
|
4141
|
+
|
|
4142
|
+
#271 M4 (spec §20): the CURRENT week is no longer re-folded from scratch each
|
|
4143
|
+
warm tick — it goes through the single-slot ``accumulate_projects_current_week``
|
|
4144
|
+
accumulator, which folds only the changed-row delta (or finalizes the cached
|
|
4145
|
+
running ``mut`` unchanged on an empty-delta tick), byte-identically.
|
|
4146
|
+
``cur_max_id`` is the tick's ``MAX(session_entries.id)`` (the regression
|
|
4147
|
+
backstop + pre-existing-row cold-refold trigger); ``cur_max_seq`` is the
|
|
4148
|
+
tick's ``MAX(mutation_seq)`` (the #270 §8 delta watermark, so an id-stable
|
|
4149
|
+
in-place finalization is caught). The accumulator is single-writer here (this
|
|
4150
|
+
runs only on the sync-thread ``use_projects_env_cache=True`` path).
|
|
3701
4151
|
"""
|
|
3702
4152
|
c = _cctally()
|
|
3703
4153
|
sc = c._load_sibling("_lib_snapshot_cache")
|
|
@@ -3730,10 +4180,45 @@ def _assemble_projects_via_cache(
|
|
|
3730
4180
|
best_order[bp] = cand
|
|
3731
4181
|
key_by_bucket[bp] = wb.first_key
|
|
3732
4182
|
|
|
4183
|
+
def _fetch_all_raw():
|
|
4184
|
+
return _aggregate_projects_week_raw(
|
|
4185
|
+
conn, week_start=cw_start, week_end=cw_end,
|
|
4186
|
+
resolver_cache=resolver_cache,
|
|
4187
|
+
)
|
|
4188
|
+
|
|
4189
|
+
def _fetch_delta_rows(after_seq):
|
|
4190
|
+
# Index-seek the changed-row delta (mutation_seq > after_seq, #270 §8)
|
|
4191
|
+
# over the current-week window, then PRE-FILTER to genuine current-week
|
|
4192
|
+
# non-synthetic rows (mirrors _fold_projects_entry's membership filter) so
|
|
4193
|
+
# the accumulator's fold-order gate compares a REAL current-week entry as
|
|
4194
|
+
# rows[0]. Sort by (ts_iso, id) to reproduce the full pass's fold order
|
|
4195
|
+
# exactly (SQLite BINARY collation on the Z-normalized ISO string ==
|
|
4196
|
+
# Python str compare).
|
|
4197
|
+
out = []
|
|
4198
|
+
for r in _projects_iter_session_entries(
|
|
4199
|
+
conn, since=cw_start, until=cw_end, after_seq=after_seq,
|
|
4200
|
+
):
|
|
4201
|
+
if r[2] == "<synthetic>": # r[2] = model
|
|
4202
|
+
continue
|
|
4203
|
+
ts = parse_iso_datetime(r[1], "session_entries.timestamp_utc")
|
|
4204
|
+
if _projects_week_start_monday_utc(ts) != cw_start:
|
|
4205
|
+
continue
|
|
4206
|
+
out.append(r)
|
|
4207
|
+
out.sort(key=lambda r: (r[1], r[0])) # (ts_iso, id)
|
|
4208
|
+
return out
|
|
4209
|
+
|
|
3733
4210
|
for w in weeks_full:
|
|
3734
4211
|
if w == cw_start:
|
|
3735
|
-
week_buckets, week_total =
|
|
3736
|
-
|
|
4212
|
+
week_buckets, week_total = sc.accumulate_projects_current_week(
|
|
4213
|
+
week_key=sc.projects_env_week_key(cw_start),
|
|
4214
|
+
cur_max_id=cur_max_id,
|
|
4215
|
+
cur_max_seq=cur_max_seq,
|
|
4216
|
+
fetch_all_raw=_fetch_all_raw,
|
|
4217
|
+
fetch_delta_rows=_fetch_delta_rows,
|
|
4218
|
+
finalize=_finalize_projects_mut,
|
|
4219
|
+
fold=lambda mut, row: _fold_projects_entry(
|
|
4220
|
+
mut, row, resolver_cache=resolver_cache, week_start=cw_start,
|
|
4221
|
+
),
|
|
3737
4222
|
)
|
|
3738
4223
|
else:
|
|
3739
4224
|
week_iso = sc.projects_env_week_key(w)
|
|
@@ -3777,7 +4262,9 @@ def _build_projects_envelope(
|
|
|
3777
4262
|
|
|
3778
4263
|
Determinism: same conn + same ``now_utc`` ⇒ byte-identical JSON
|
|
3779
4264
|
(R-PROJ5 invariant). Per-tick memoized on
|
|
3780
|
-
``(max(session_entries.id),
|
|
4265
|
+
``(max(session_entries.id), max_wus_id, max(mutation_seq), cw_week_start,
|
|
4266
|
+
weeks_back)`` — the ``mutation_seq`` leg (#270 §7d) busts the memo on an
|
|
4267
|
+
id-stable in-place finalization.
|
|
3781
4268
|
"""
|
|
3782
4269
|
c = _cctally()
|
|
3783
4270
|
|
|
@@ -3797,10 +4284,30 @@ def _build_projects_envelope(
|
|
|
3797
4284
|
max_wus_id = cur.fetchone()[0]
|
|
3798
4285
|
except sqlite3.OperationalError:
|
|
3799
4286
|
max_wus_id = 0
|
|
4287
|
+
# #270 (§7d, Codex-2b): this OUTER whole-envelope memo returns BEFORE the
|
|
4288
|
+
# inner per-(project, week) reconcile cache, so an id-stable in-place
|
|
4289
|
+
# finalization — which leaves MAX(id)/MAX(wus id) flat — would serve the
|
|
4290
|
+
# stale envelope regardless of the inner seq fix. Fold the mutation signal
|
|
4291
|
+
# into the memo key so it busts on any changed row. `MAX(mutation_seq)` is
|
|
4292
|
+
# read off the SAME `session_entries` surface as `max_id` (an id-stable
|
|
4293
|
+
# UPSERT advances it), index-backed by `idx_entries_mutation_seq`, and
|
|
4294
|
+
# degrades to 0 where the column is absent (old fixtures) — byte-identical.
|
|
4295
|
+
# Deliberately `MAX(mutation_seq)`, NOT the `cache_meta` counter the reconciles
|
|
4296
|
+
# read via `_entry_mutation_seq` (this conn is a project-detail view without a
|
|
4297
|
+
# `cache_meta` surface). The two are consistent — the counter is always
|
|
4298
|
+
# >= MAX(mutation_seq), and no row exists strictly between them at read time —
|
|
4299
|
+
# so do not "unify" them here (the divergence is intentional; see #270 §M3-1).
|
|
4300
|
+
try:
|
|
4301
|
+
cur = conn.execute(
|
|
4302
|
+
"SELECT COALESCE(MAX(mutation_seq), 0) FROM session_entries"
|
|
4303
|
+
)
|
|
4304
|
+
entry_mutation_seq = cur.fetchone()[0]
|
|
4305
|
+
except sqlite3.OperationalError:
|
|
4306
|
+
entry_mutation_seq = 0
|
|
3800
4307
|
cw_key: "dt.datetime | None" = None
|
|
3801
4308
|
if current_week is not None:
|
|
3802
4309
|
cw_key = getattr(current_week, "week_start_at", None)
|
|
3803
|
-
memo_key = (max_id, max_wus_id, cw_key, weeks_back)
|
|
4310
|
+
memo_key = (max_id, max_wus_id, entry_mutation_seq, cw_key, weeks_back)
|
|
3804
4311
|
cached = _PROJECTS_ENV_MEMO.get("value")
|
|
3805
4312
|
if cached is not None and _PROJECTS_ENV_MEMO.get("key") == memo_key:
|
|
3806
4313
|
return cached
|
|
@@ -3848,6 +4355,7 @@ def _build_projects_envelope(
|
|
|
3848
4355
|
if use_projects_env_cache:
|
|
3849
4356
|
buckets, total_cost_by_week, key_by_bucket = _assemble_projects_via_cache(
|
|
3850
4357
|
conn, weeks_full=weeks_full, cw_start=cw_start, cw_end=cw_end,
|
|
4358
|
+
cur_max_id=max_id, cur_max_seq=entry_mutation_seq,
|
|
3851
4359
|
)
|
|
3852
4360
|
else:
|
|
3853
4361
|
# `_resolve_project_key` is the production resolver; we use git-root
|