cctally 1.108.0 → 1.109.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/CHANGELOG.md +101 -0
  2. package/bin/_cctally_account.py +4 -0
  3. package/bin/_cctally_alerts.py +6 -2
  4. package/bin/_cctally_cache.py +1590 -153
  5. package/bin/_cctally_config.py +3 -2
  6. package/bin/_cctally_core.py +589 -32
  7. package/bin/_cctally_dashboard.py +1223 -160
  8. package/bin/_cctally_dashboard_conversation.py +263 -8
  9. package/bin/_cctally_dashboard_envelope.py +144 -4
  10. package/bin/_cctally_dashboard_share.py +154 -36
  11. package/bin/_cctally_dashboard_sources.py +2146 -347
  12. package/bin/_cctally_db.py +634 -5
  13. package/bin/_cctally_diagnosis_sources.py +57 -11
  14. package/bin/_cctally_doctor.py +84 -15
  15. package/bin/_cctally_five_hour.py +1295 -40
  16. package/bin/_cctally_forecast.py +109 -26
  17. package/bin/_cctally_journal.py +380 -37
  18. package/bin/_cctally_milestone_history.py +86 -6
  19. package/bin/_cctally_percent_breakdown.py +6 -1
  20. package/bin/_cctally_project.py +48 -76
  21. package/bin/_cctally_quota.py +39 -7
  22. package/bin/_cctally_quota_model.py +17 -2
  23. package/bin/_cctally_record.py +1603 -437
  24. package/bin/_cctally_refresh.py +2 -1
  25. package/bin/_cctally_share.py +24 -5
  26. package/bin/_cctally_source_analytics.py +553 -30
  27. package/bin/_cctally_statusline.py +83 -19
  28. package/bin/_cctally_store.py +61 -4
  29. package/bin/_cctally_tui.py +229 -74
  30. package/bin/_cctally_weekrefs.py +28 -5
  31. package/bin/_lib_blocks.py +127 -20
  32. package/bin/_lib_cache_report.py +20 -5
  33. package/bin/_lib_codex_conversation_query.py +11 -1
  34. package/bin/_lib_codex_hooks.py +102 -0
  35. package/bin/_lib_conversation.py +119 -0
  36. package/bin/_lib_conversation_dispatch.py +31 -1
  37. package/bin/_lib_conversation_query.py +61 -14
  38. package/bin/_lib_conversation_retention.py +413 -25
  39. package/bin/_lib_dashboard_sources.py +267 -7
  40. package/bin/_lib_diagnosis.py +10 -0
  41. package/bin/_lib_diff_kernel.py +12 -1
  42. package/bin/_lib_doctor.py +214 -6
  43. package/bin/_lib_ingest_frontier.py +1366 -49
  44. package/bin/_lib_journal.py +19 -2
  45. package/bin/_lib_merge_gate.py +307 -0
  46. package/bin/_lib_pricing.py +522 -18
  47. package/bin/_lib_record.py +130 -12
  48. package/bin/_lib_render.py +9 -3
  49. package/bin/_lib_retained_size.py +27 -2
  50. package/bin/_lib_segment_summary.py +66 -8
  51. package/bin/_lib_share_templates.py +28 -18
  52. package/bin/_lib_snapshot_cache.py +115 -1
  53. package/bin/_lib_source_retry.py +320 -0
  54. package/bin/_lib_statusline_candidates.py +400 -36
  55. package/bin/_lib_tick_stats.py +119 -0
  56. package/bin/_lib_view_models.py +251 -76
  57. package/bin/cctally +69 -0
  58. package/dashboard/static/assets/ConversationsView-TO5bpIyc.js +72 -0
  59. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-BZGXOlc2.js} +1 -1
  60. package/dashboard/static/assets/ModalRoot-C_fvugTl.js +1 -0
  61. package/dashboard/static/assets/{ProjectsDrillPanel-QLI9i5mZ.js → ProjectsDrillPanel-KbO_KWa7.js} +1 -1
  62. package/dashboard/static/assets/SourceDetailModal-BrOPLVRL.js +1 -0
  63. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-CYgEFAi4.js} +2 -2
  64. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  65. package/dashboard/static/assets/index-Cj_l7T7u.css +1 -0
  66. package/dashboard/static/assets/index-DLVwACUM.js +13 -0
  67. package/dashboard/static/assets/outlineNavigation-DXX-RjKl.js +9 -0
  68. package/dashboard/static/assets/useKeymap-DQx_A1MB.js +1 -0
  69. package/dashboard/static/dashboard.html +3 -3
  70. package/package.json +3 -1
  71. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  72. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  73. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  74. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  75. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  76. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  77. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  78. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -436,7 +436,36 @@ _init_paths_from_env()
436
436
  # is a bump and not a migration is unchanged: the registry is frozen at 13,
437
437
  # and an epoch-current open returns before any schema work, so a handler or an
438
438
  # `add_column_if_missing` would never run on an upgraded install.
439
- STATS_INDEX_EPOCH = 1013
439
+ # 1013 -> 1014 (#769 S2 §3): source-local five-hour credit confirmation state.
440
+ # Adds `five_hour_credit_confirmation_state`, keyed
441
+ # `(account_key, five_hour_window_key, source)`. Five-hour credit detection
442
+ # used to compare an incoming reading against the latest accepted snapshot
443
+ # whatever contributor produced it, so one stale `source=statusline` sample
444
+ # below an `source=api` baseline fabricated a credit with no confirmation from
445
+ # any source (the 2026-09-04 incident, #751). The state carries each
446
+ # contributor's own pre-drop baseline and its pending descent, so a descent
447
+ # observed by one source can only be confirmed by a later, distinct
448
+ # observation from that same source. It is disposable operational state rather
449
+ # than journal truth — the rebuild does not reproduce it and must not count it
450
+ # — and the cold-start consequence of losing it is stated at the helpers in
451
+ # `bin/_cctally_record.py`. The mechanical reason this is a bump and not a
452
+ # migration is unchanged: the registry is frozen at 13, and an epoch-current
453
+ # open returns before any schema work, so a handler or an
454
+ # `add_column_if_missing` would never run on an upgraded install.
455
+ # 1014 -> 1015 (#769 S11, issue #824): weekly_usage_snapshots gains
456
+ # `weekly_observation_held`, which separates a weekly value that was genuinely
457
+ # observed on this tick from one carried forward so a weekly-clamped tick can
458
+ # still persist its five-hour evidence. Before it, a tick whose weekly percent
459
+ # clamped against the stored high-water mark wrote no row at all, so the same
460
+ # tick's genuine five-hour reading was discarded — unclamped and unpersisted.
461
+ # A held row (`weekly_observation_held = 1`) carries the tick's own capture
462
+ # time, source and five-hour fields, while its weekly value and boundary are
463
+ # copied from the latest non-held basis. Weekly readers exclude held rows;
464
+ # five-hour readers include them. The mechanical reason this is a bump and not
465
+ # a migration is unchanged: the registry is frozen at 13, and an epoch-current
466
+ # open returns before any schema work, so a handler or an
467
+ # `add_column_if_missing` would never run on an upgraded install.
468
+ STATS_INDEX_EPOCH = 1015
440
469
  LEGACY_STATS_HEAD = 13
441
470
 
442
471
  #: #496 S1 F1. A NEW branch, for a state that cannot occur before the
@@ -2102,7 +2131,9 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2102
2131
  page_url TEXT,
2103
2132
  source TEXT NOT NULL DEFAULT 'userscript',
2104
2133
  payload_json TEXT NOT NULL,
2105
- account_key TEXT NOT NULL DEFAULT 'unattributed'
2134
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
2135
+ weekly_observation_held INTEGER NOT NULL DEFAULT 0
2136
+ CHECK (weekly_observation_held IN (0, 1))
2106
2137
  )
2107
2138
  """
2108
2139
  )
@@ -2148,6 +2179,19 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2148
2179
  add_column_if_missing(
2149
2180
  conn, "weekly_usage_snapshots", "account_key",
2150
2181
  "TEXT NOT NULL DEFAULT 'unattributed'")
2182
+ # weekly_observation_held (#769 S11, issue #824): the provenance flag
2183
+ # rides the STATS_INDEX_EPOCH 1015 bump, so a fresh rebuild carries it
2184
+ # via the CREATE TABLE above. This backstop keeps an already-1015 index
2185
+ # that predates the column consistent, and covers the interval between
2186
+ # a new binary's first open of an epoch-1014 store and the background
2187
+ # rebuild that resolves the mismatch. Every production writer passes the
2188
+ # flag explicitly; the DEFAULT 0 is the rev-4.1 defensive backstop and
2189
+ # is also the correct value for every row written before the column
2190
+ # existed, because those rows are all genuine weekly observations.
2191
+ add_column_if_missing(
2192
+ conn, "weekly_usage_snapshots", "weekly_observation_held",
2193
+ "INTEGER NOT NULL DEFAULT 0 "
2194
+ "CHECK (weekly_observation_held IN (0, 1))")
2151
2195
  add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
2152
2196
  add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
2153
2197
  # five_hour_window_key — canonical (10-min-floored epoch) key for
@@ -2283,6 +2327,24 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2283
2327
  # tuple. One index cannot express both, and a single index over the
2284
2328
  # tuple would refuse a second genuine reset that happens to share a
2285
2329
  # boundary pair with the first.
2330
+ # `observed_pre_credit_pct` is a RETAINED NON-INPUT (#761 residual 4,
2331
+ # decision D2). The STORED COLUMN records what the pre-credit reading
2332
+ # was and arms nothing: no read path branches on it, and the clamp floor
2333
+ # that does arm behaviour lives in `weekly_credit_floors`. It is kept,
2334
+ # with its journal payload field, because removing either would break
2335
+ # journal compatibility for no gain, and because it is the only record
2336
+ # of the baseline a historical reset was measured against.
2337
+ #
2338
+ # Do not confuse it with the LIVE FUNCTION ARGUMENT of the same name.
2339
+ # `_fire_in_place_credit(..., observed_pre_credit_pct=...)` is a real
2340
+ # input: it is the band the stale-replica DELETE and its two doomed
2341
+ # captures select on (`_stale_replica_band_sql`). The argument decides
2342
+ # which rows are removed; the column decides nothing.
2343
+ #
2344
+ # This note stays OUTSIDE the SQL. SQLite retains a `CREATE TABLE`
2345
+ # statement verbatim in `sqlite_master.sql`, and `_REBUILD_SCHEMA_
2346
+ # FINGERPRINT` digests that text, so an inline `--` comment here fails
2347
+ # every rebuild with a schema definition mismatch.
2286
2348
  conn.execute(
2287
2349
  """
2288
2350
  CREATE TABLE IF NOT EXISTS week_reset_events (
@@ -2342,6 +2404,39 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2342
2404
  )
2343
2405
  """
2344
2406
  )
2407
+ # #769 S2 §3 epoch 1014: source-local five-hour credit confirmation
2408
+ # state. One row per `(account_key, five_hour_window_key, source)`.
2409
+ #
2410
+ # `baseline_pct` is that contributor's own running maximum inside the
2411
+ # window, held raw — never read back from `weekly_usage_snapshots`,
2412
+ # whose five-hour percent is MAX-clamped at write time and therefore
2413
+ # sits downstream of the defect this table exists to remove.
2414
+ #
2415
+ # The four `pending_*` columns are the armed descent, and they are
2416
+ # nullable together: `pending_low_pct IS NOT NULL` is the armed
2417
+ # predicate. `pending_observation_id` is the arming observation's raw
2418
+ # journal id, so a byte-identical replay of that line cannot confirm
2419
+ # itself; `pending_at_utc` is the arming tick's detection clock, which
2420
+ # becomes the credit's effective instant on confirm.
2421
+ #
2422
+ # Disposable operational state, like `weekly_reset_debounce_state`
2423
+ # above: a rebuild legitimately loses it, and the cold-start
2424
+ # consequence is stated at `_read_five_hour_source_state` in
2425
+ # `bin/_cctally_record.py`.
2426
+ conn.execute(
2427
+ """
2428
+ CREATE TABLE IF NOT EXISTS five_hour_credit_confirmation_state (
2429
+ account_key TEXT NOT NULL,
2430
+ five_hour_window_key INTEGER NOT NULL,
2431
+ source TEXT NOT NULL,
2432
+ baseline_pct REAL NOT NULL,
2433
+ pending_low_pct REAL,
2434
+ pending_at_utc TEXT,
2435
+ pending_observation_id TEXT,
2436
+ PRIMARY KEY (account_key, five_hour_window_key, source)
2437
+ )
2438
+ """
2439
+ )
2345
2440
  _backfill_week_reset_events(conn)
2346
2441
 
2347
2442
  # ── five_hour_reset_events (Anthropic-issued in-place 5h credits) ──
@@ -3202,36 +3297,88 @@ def _get_latest_row_for_week(
3202
3297
  as_of_utc: str | None = None,
3203
3298
  *,
3204
3299
  account_key: str | None = None,
3300
+ since_utc: str | None = None,
3301
+ before_utc: str | None = None,
3302
+ exclude_held: bool = False,
3205
3303
  ) -> sqlite3.Row | None:
3206
3304
  """Latest week row from a weekly snapshot table.
3207
3305
 
3306
+ ``exclude_held`` (#769 S11, #824) drops `weekly_observation_held = 1` rows.
3307
+ It is bound to the CALLER rather than applied unconditionally because this
3308
+ primitive serves two tables: `weekly_usage_snapshots` has the column and
3309
+ `weekly_cost_snapshots` does not, so an unconditional predicate would make
3310
+ every cost lookup raise. `get_latest_usage_for_week` passes it; the cost
3311
+ wrapper in `bin/_cctally_milestones.py` does not.
3312
+
3208
3313
  ``account_key`` (#341): ``None`` = the account-blind merged read (today's
3209
3314
  behavior, byte-stable); a real key scopes to that account's rows — the
3210
3315
  ``--account`` render consumers and the write-path milestone-cost read (P2-1)
3211
- pass it explicitly."""
3212
- acct_pred = "" if account_key is None else " AND account_key = ?"
3213
- acct_params: tuple = () if account_key is None else (account_key,)
3214
- if as_of_utc is None:
3215
- return conn.execute(
3216
- f"""
3217
- SELECT *
3218
- FROM {table_name}
3219
- WHERE week_start_date = ?{acct_pred}
3220
- ORDER BY captured_at_utc DESC, id DESC
3221
- LIMIT 1
3222
- """,
3223
- (week_ref.week_start.isoformat(),) + acct_params,
3224
- ).fetchone()
3316
+ pass it explicitly.
3317
+
3318
+ ``since_utc`` / ``before_utc`` (#750 S4 spec §1.2) bound the read to ONE
3319
+ billing cycle of a credited week: ``since_utc`` is INCLUSIVE and
3320
+ ``before_utc`` is EXCLUSIVE, which is `_aggregate_weekly`'s
3321
+ ``[start_ts, end_ts)`` convention, so a snapshot captured exactly at a
3322
+ credit instant lands in the same segment as an entry stamped there.
3323
+
3324
+ ``as_of_utc`` keeps its inclusive ``<=`` unchanged. It is a different
3325
+ quantity — the caller's global freshness bound — and other callers reach
3326
+ this primitive with it and no segment bounds
3327
+ (`bin/_cctally_sync_week.py`, and `get_latest_cost_for_week` in
3328
+ `bin/_cctally_milestones.py`, which reaches it for the cost table).
3329
+ Redefining it would move them."""
3330
+ preds = ["week_start_date = ?"]
3331
+ params: list = [week_ref.week_start.isoformat()]
3332
+ if as_of_utc is not None:
3333
+ # `unixepoch(...)` here too, for the reason the `since_utc` comment
3334
+ # below gives: a lexical compare disagrees with an instant compare on
3335
+ # two spellings of one moment, and the two consumers of this table
3336
+ # would then read different rows. `captured_at_utc` is stamped at
3337
+ # seconds precision (`now_utc_iso`), so the conversion loses nothing.
3338
+ preds.append("unixepoch(captured_at_utc) <= unixepoch(?)")
3339
+ params.append(as_of_utc)
3340
+ # One consequence, stated because a sibling reader of this table
3341
+ # disagrees with it on purpose: `unixepoch()` is NULL on an
3342
+ # unparseable `captured_at_utc`, so such a row fails this predicate
3343
+ # and sorts LAST under the DESC below. It is excluded here, and
3344
+ # `latest_usage_by_segment` excludes it too, because both answer which
3345
+ # single row a cycle owns and a row whose instant is unknown cannot be
3346
+ # placed in one. `_floored_week_max` RETAINS it, because it answers a
3347
+ # week's maximum and dropping a row would understate that. Neither is
3348
+ # a bug; the reasons are recorded at both sites.
3349
+ if since_utc is not None:
3350
+ # INCLUSIVE lower: the cut instant belongs to the segment that STARTS
3351
+ # there (spec §1.2). Without it a credited tail with no post-cut
3352
+ # capture returns the latest PRE-cut reading — a percent from the
3353
+ # previous cycle — instead of being reported as missing.
3354
+ #
3355
+ # `unixepoch(...)`, never a lexical compare. `captured_at_utc` is
3356
+ # written in more than one offset spelling across the tree, and
3357
+ # `'…+00:00' < '…Z'` is TRUE for one instant written two ways: a
3358
+ # capture stamped exactly at a credit would then fall in the
3359
+ # PREDECESSOR, which is precisely the ownership rule §1.2 fixes. The
3360
+ # same defence `_reset_aware_floor` and `_latest_reset_event_for_end`
3361
+ # already document.
3362
+ preds.append("unixepoch(captured_at_utc) >= unixepoch(?)")
3363
+ params.append(since_utc)
3364
+ if before_utc is not None:
3365
+ # EXCLUSIVE upper: mirrors `_aggregate_weekly`'s [start, end).
3366
+ preds.append("unixepoch(captured_at_utc) < unixepoch(?)")
3367
+ params.append(before_utc)
3368
+ if account_key is not None:
3369
+ preds.append("account_key = ?")
3370
+ params.append(account_key)
3371
+ if exclude_held:
3372
+ preds.append("weekly_observation_held = 0")
3373
+ # Ordered by the PARSED instant, matching how `latest_usage_by_segment`
3374
+ # ranks the same rows. A lexical `captured_at_utc DESC` and an instant
3375
+ # ranking disagree on two spellings of one moment, so the two consumers of
3376
+ # `weekly_usage_snapshots` could pick different rows for one segment. `id`
3377
+ # stays the deterministic tie-breaker, which it already was.
3225
3378
  return conn.execute(
3226
- f"""
3227
- SELECT *
3228
- FROM {table_name}
3229
- WHERE week_start_date = ?
3230
- AND captured_at_utc <= ?{acct_pred}
3231
- ORDER BY captured_at_utc DESC, id DESC
3232
- LIMIT 1
3233
- """,
3234
- (week_ref.week_start.isoformat(), as_of_utc) + acct_params,
3379
+ f"SELECT * FROM {table_name} WHERE " + " AND ".join(preds)
3380
+ + " ORDER BY unixepoch(captured_at_utc) DESC, id DESC LIMIT 1",
3381
+ tuple(params),
3235
3382
  ).fetchone()
3236
3383
 
3237
3384
 
@@ -3352,6 +3499,384 @@ def _ordered_in_place_cuts(
3352
3499
  return ordered
3353
3500
 
3354
3501
 
3502
+ def in_place_cut_instants(
3503
+ conn: sqlite3.Connection, *, account_key: "str | None",
3504
+ ) -> "frozenset[dt.datetime]":
3505
+ """This account's admitted in-place cut instants, UTC-aware.
3506
+
3507
+ #750 S4 spec §1.1. The consumer-side twin of what both appliers decide
3508
+ internally, so `build_weekly_view` and `build_trend_view` cannot drift
3509
+ apart from each other or from the appliers (invariant 1). Five details
3510
+ make it equivalent to them, and each one decides a real case:
3511
+
3512
+ * the in-place TEST is RAW string equality on the unparsed columns,
3513
+ matching `_apply_reset_events_to_subweeks`
3514
+ (bin/_lib_subscription_weeks.py) and `_apply_reset_events_to_weekrefs`
3515
+ (bin/_cctally_weekrefs.py). NOT `unixepoch()`, and NOT
3516
+ `_LEGACY_IN_PLACE_SHAPE` — that predicate is for legacy-row detection
3517
+ and additionally admits the pre-v1.7.2 `(cur_end, cur_end)` shape,
3518
+ which the appliers do not split. A canonicalizing test would bound
3519
+ segments neither applier cut, on rows whose two columns spell one
3520
+ instant in two offsets;
3521
+ * the per-week grouping parses `new_week_end_at`, because that column
3522
+ carries mixed offset spellings, and it is what bounds each cut against
3523
+ its own week's end;
3524
+ * the read is account-scoped, exactly as both appliers scope theirs;
3525
+ * cut instants are deduplicated BY INSTANT and kept unrounded;
3526
+ * a cut at or after its week's end, or at or before a boundary shift
3527
+ that moved that week's start, is rejected through the same
3528
+ `_ordered_in_place_cuts` both appliers call.
3529
+
3530
+ The rows are ordered the way `_latest_reset_event_for_end` orders, and
3531
+ the boundary-shift lower bound is taken with `setdefault`, because that
3532
+ is how both appliers pick the single row that moves a week's start.
3533
+
3534
+ The RESULT is flat rather than keyed by week, because no consumer can
3535
+ reliably name a week's end: `_apply_overlap_clamp_to_subweeks` and a
3536
+ boundary-shift event both move a tail's end away from
3537
+ `new_week_end_at` after the appliers have run. `credited_segment_cuts`
3538
+ is what binds these instants to one week's segments, and it also supplies
3539
+ the remaining lower bound — the week's own start on an unshifted week,
3540
+ which `week_reset_events` does not record.
3541
+
3542
+ An absent `week_reset_events` table yields an empty set: a render-path
3543
+ consumer of an optional table degrades to "this install recorded no
3544
+ credit" rather than raising.
3545
+ """
3546
+ have_table = conn.execute(
3547
+ "SELECT 1 FROM sqlite_master "
3548
+ "WHERE type IN ('table','view') AND name = 'week_reset_events'"
3549
+ ).fetchone()
3550
+ if have_table is None:
3551
+ return frozenset()
3552
+ acct_pred = "" if account_key is None else " WHERE account_key = ?"
3553
+ acct_params: tuple = () if account_key is None else (account_key,)
3554
+ try:
3555
+ rows = conn.execute(
3556
+ "SELECT old_week_end_at, new_week_end_at, effective_reset_at_utc "
3557
+ "FROM week_reset_events" + acct_pred
3558
+ + " ORDER BY unixepoch(effective_reset_at_utc) DESC, id DESC",
3559
+ acct_params,
3560
+ ).fetchall()
3561
+ except sqlite3.OperationalError:
3562
+ # A stats.db predating the `account_key` column (#620 S1): an install
3563
+ # whose migrations have not run, or a dev binary that refused the prod
3564
+ # forward-migration (#142). Withholding is the degradation, exactly as
3565
+ # `_projects_week_grid` withholds on the same predicate — never a
3566
+ # retry without the scope, which would let one account's credit bound
3567
+ # another account's segment.
3568
+ return frozenset()
3569
+ # `post_map` is the boundary shift that moves a week's START; it is the
3570
+ # lower bound every cut of that week must fall after. Only a NON-in-place
3571
+ # row plays that role, which is the same split both appliers make.
3572
+ post_map: dict[dt.datetime, str] = {}
3573
+ cuts_raw: dict[dt.datetime, list[tuple[dt.datetime, str]]] = {}
3574
+ for r in rows:
3575
+ raw_eff = r["effective_reset_at_utc"]
3576
+ try:
3577
+ new_dt = parse_iso_datetime(r["new_week_end_at"], "evt.new_end")
3578
+ except (TypeError, ValueError):
3579
+ continue
3580
+ if r["old_week_end_at"] == raw_eff:
3581
+ try:
3582
+ eff_dt = parse_iso_datetime(raw_eff, "evt.eff")
3583
+ except (TypeError, ValueError):
3584
+ continue
3585
+ cuts_raw.setdefault(new_dt, []).append((eff_dt, raw_eff))
3586
+ else:
3587
+ post_map.setdefault(new_dt, raw_eff)
3588
+ out: set[dt.datetime] = set()
3589
+ for end_dt, cuts in cuts_raw.items():
3590
+ for cut_dt, _raw in _ordered_in_place_cuts(
3591
+ cuts, post_map.get(end_dt), end_dt):
3592
+ out.add(cut_dt.astimezone(dt.timezone.utc))
3593
+ return frozenset(out)
3594
+
3595
+
3596
+ def credited_segment_cuts(
3597
+ segment_starts, cut_instants,
3598
+ ) -> "frozenset[dt.datetime]":
3599
+ """The cut instants that bound ONE week's emitted segments (#750 S4 §1.1).
3600
+
3601
+ ``segment_starts`` are the post-applier start instants of every segment
3602
+ sharing one week identity — `SubWeek.start_date` for the subweeks
3603
+ consumers, `WeekRef.key` for the trend one. ``cut_instants`` is
3604
+ `in_place_cut_instants`. The result is empty for an uncredited week, so
3605
+ ``bool(...)`` is the credited predicate and membership is the per-segment
3606
+ bounded predicate.
3607
+
3608
+ A week's segments are matched to their cuts by their STARTS, never by the
3609
+ week's end. Both appliers emit ``[start, c1)``, each ``[ci, ci+1)`` and
3610
+ ``[cn, end)``, so every cut is verbatim the start of the segment that
3611
+ follows it, while the LAST segment's end is the week end — and two
3612
+ ordinary mechanisms move that end away from the `new_week_end_at` the
3613
+ events table records. `_apply_overlap_clamp_to_subweeks` runs after the
3614
+ reset applier and pulls a tail's end back to the successor's anchor when
3615
+ that anchor drifted earlier, and a boundary-shift event whose
3616
+ `old_week_end_at` is this week's end rewrites the end to the shift
3617
+ instant. Keyed on the week end, the lookup then matched nothing, the week
3618
+ read as uncredited, and BOTH cycles resolved to the week's latest
3619
+ observation — the post-credit reading on the pre-credit cycle, which is
3620
+ #731 / #736 back in full, and silently.
3621
+
3622
+ The EARLIEST start is excluded, which is not a formality: it is the
3623
+ week's own effective start, and both appliers reject a cut at or before
3624
+ it (`_ordered_in_place_cuts`, called with the boundary shift when the
3625
+ week carries one and with the week's own start when it does not). The
3626
+ helper cannot apply that lower bound itself, because `week_reset_events`
3627
+ records no week start; here the segments supply it, so the rule holds on
3628
+ a shifted and an unshifted week alike.
3629
+ """
3630
+ ordered = sorted({s for s in segment_starts if s is not None})
3631
+ return frozenset(s for s in ordered[1:] if s in cut_instants)
3632
+
3633
+
3634
+ def canonical_capture_bound(value_dt: "dt.datetime") -> str:
3635
+ """A segment bound spelled the way ``captured_at_utc`` is stored.
3636
+
3637
+ UTC, whole seconds, ``Z``. SQLite compares those lexicographically, so a
3638
+ bound written in another offset would order wrongly against the stored
3639
+ column. One definition, shared by every consumer that bounds a segment.
3640
+ """
3641
+ return (
3642
+ value_dt.astimezone(dt.timezone.utc)
3643
+ .replace(microsecond=0)
3644
+ .isoformat()
3645
+ .replace("+00:00", "Z")
3646
+ )
3647
+
3648
+
3649
+ def _segment_instant(value: "str | None") -> "dt.datetime | None":
3650
+ if not value:
3651
+ return None
3652
+ try:
3653
+ return parse_iso_datetime(
3654
+ value, "subweek.bound").astimezone(dt.timezone.utc)
3655
+ except (TypeError, ValueError):
3656
+ return None
3657
+
3658
+
3659
+ def segment_capture_bounds(
3660
+ conn: sqlite3.Connection, subweeks, *, account_key: "str | None",
3661
+ ) -> "dict[str, tuple[str | None, str | None]]":
3662
+ """``{SubWeek.segment_key -> (since_utc, before_utc)}`` (#750 S4 §2.1).
3663
+
3664
+ ``since_utc`` is INCLUSIVE and ``before_utc`` EXCLUSIVE, which is
3665
+ `_aggregate_weekly`'s ``[start, end)``, so an observation captured exactly
3666
+ at a credit instant lands in the same cycle as an entry stamped there.
3667
+
3668
+ Only a segment whose END is an in-place cut carries an upper bound. The
3669
+ lower bound is the LATER of two things, and is ``None`` when neither
3670
+ applies:
3671
+
3672
+ * the segment's own start, when its week was credited. That is §1.2's
3673
+ rule, and on the tail it is what makes a cycle with no post-cut
3674
+ observation report as MISSING instead of resolving to the previous
3675
+ cycle's reading;
3676
+ * the latest `week_reset_events` / `weekly_credit_floors` instant lying
3677
+ inside this segment's own interval. That is `_reset_aware_floor`, the
3678
+ chokepoint every other clamp site consults, called per SEGMENT rather
3679
+ than per week. Scoped to the week, its
3680
+ automatic leg discarded pre-credit captures for BOTH segments and so
3681
+ erased the very pre-credit reading it was meant to preserve; scoped to
3682
+ the segment it lands on the tail alone, where it agrees with the first
3683
+ clause. On a boundary-shifted week it still floors at the shift, and a
3684
+ manual `record-credit` floor — written under the same-window model,
3685
+ which re-anchors nothing and creates no segment — still raises the
3686
+ bound of the segment it falls in.
3687
+
3688
+ A segment of a plain uncredited week therefore has NO lower bound, which
3689
+ is what `weekly` does. Imposing one would discard an observation captured
3690
+ a little BEFORE its week's anchor — the mirror of the post-boundary
3691
+ capture jitter the weekly surfaces already tolerate, and a real shape in
3692
+ the fixtures.
3693
+ """
3694
+ segments = list(subweeks)
3695
+ if not segments:
3696
+ return {}
3697
+ cut_instants = in_place_cut_instants(conn, account_key=account_key)
3698
+ by_date: dict = {}
3699
+ for sw in segments:
3700
+ by_date.setdefault(sw.start_date.isoformat(), []).append(sw)
3701
+
3702
+ out: dict = {}
3703
+ for _date_key, group in by_date.items():
3704
+ # Matched by the segments' own STARTS — a cut is verbatim the start
3705
+ # of the segment that follows it, while a tail's END can be moved
3706
+ # after the appliers run by the overlap clamp or a boundary shift.
3707
+ week_cuts = credited_segment_cuts(
3708
+ (_segment_instant(sw.start_ts) for sw in group), cut_instants)
3709
+ credited = bool(week_cuts)
3710
+ for sw in group:
3711
+ start_dt = _segment_instant(sw.start_ts)
3712
+ end_dt = _segment_instant(sw.end_ts)
3713
+ lower = start_dt if (credited and start_dt is not None) else None
3714
+ # The floor comes from the SAME chokepoint every other clamp site
3715
+ # consults, called with this SEGMENT's bounds rather than the
3716
+ # week's. Scoped to the week, its reset leg discarded pre-credit
3717
+ # captures for both cycles; scoped to the segment it lands on the
3718
+ # tail alone. The returned instant is re-checked against the
3719
+ # segment interval because the manual `weekly_credit_floors` leg
3720
+ # is keyed on the week's date and carries no interval bound of
3721
+ # its own.
3722
+ try:
3723
+ floor_dt = _segment_instant(_reset_aware_floor(
3724
+ conn, sw.start_date.isoformat(), sw.start_ts, sw.end_ts,
3725
+ account_key=account_key,
3726
+ ))
3727
+ except sqlite3.OperationalError:
3728
+ # A store with neither floor table: an install whose
3729
+ # migrations have not run, or a unit fixture holding only the
3730
+ # snapshot table. The leg is absent, not wrong.
3731
+ floor_dt = None
3732
+ if floor_dt is not None:
3733
+ inside = (
3734
+ (start_dt is None or floor_dt >= start_dt)
3735
+ and (end_dt is None or floor_dt < end_dt)
3736
+ )
3737
+ if inside and (lower is None or floor_dt > lower):
3738
+ lower = floor_dt
3739
+ before = (
3740
+ canonical_capture_bound(end_dt)
3741
+ if (end_dt is not None and end_dt in week_cuts) else None
3742
+ )
3743
+ out[sw.segment_key] = (
3744
+ canonical_capture_bound(lower) if lower is not None else None,
3745
+ before,
3746
+ )
3747
+ return out
3748
+
3749
+
3750
+ def weekly_held_exclusion(conn, *, prefix: str = " AND ") -> str:
3751
+ """``AND weekly_observation_held = 0``, or empty on a store without it.
3752
+
3753
+ Omitting the predicate where the column is absent is not a compromise: the
3754
+ column arrived with epoch 1015, and a store that does not carry it cannot
3755
+ contain a held row, so the filtered and unfiltered populations are
3756
+ identical. Production never takes that branch — `open_db` raises
3757
+ `StatsEpochRebuildDeferred` on an epoch-mismatched store before any read,
3758
+ and a legacy store gains the column at the schema apply — but several
3759
+ hand-built fixture stores stop short of the current schema, and the readers
3760
+ that use this already tolerate an older shape by design.
3761
+ """
3762
+ try:
3763
+ cols = {str(row[1]) for row in
3764
+ conn.execute("PRAGMA table_info(weekly_usage_snapshots)")}
3765
+ except sqlite3.Error:
3766
+ return ""
3767
+ if "weekly_observation_held" not in cols:
3768
+ return ""
3769
+ return f"{prefix}weekly_observation_held = 0"
3770
+
3771
+
3772
+ def latest_usage_by_segment(
3773
+ conn: sqlite3.Connection, subweeks, *,
3774
+ account_key: "str | None" = None,
3775
+ as_of_utc: "str | None" = None,
3776
+ rows=None,
3777
+ ) -> "dict[str, float | None]":
3778
+ """``{SubWeek.segment_key -> the latest weekly_percent in that cycle}``.
3779
+
3780
+ #750 S4 §2.1. One value per SEGMENT identity, never per calendar date:
3781
+ `_load_week_snapshots` returned ``{week_start_utc -> max(weekly_percent)}``
3782
+ keyed on `weekly_usage_snapshots.week_start_at`, so on a credited week the
3783
+ post-credit segment matched nothing and the pre-credit one received the
3784
+ post-credit reading (#731). A segment with no observation inside its own
3785
+ interval maps to ``None`` rather than borrowing a neighbour's.
3786
+
3787
+ LATEST, not MAX. "weekly_percent is monotonic within a week" is true of a
3788
+ billing CYCLE and false of a credited week, which is why the dashboard
3789
+ projects query already reads the latest row. Taking the maximum would lock
3790
+ a credited week's attribution to the pre-credit high-water mark forever.
3791
+ Inside one cycle the two agree.
3792
+
3793
+ It takes the CALLER's connection: `_load_week_snapshots` opened its own
3794
+ while the dashboard already holds a transaction, and a reducer that opened
3795
+ its own would give the dashboard a second, possibly inconsistent read.
3796
+
3797
+ ``rows`` lets a caller that has already fetched and resolved its own rows
3798
+ hand them over as ``(segment_key, captured_at_utc, row_id,
3799
+ weekly_percent)`` tuples; the bound and latest-wins decisions still happen
3800
+ here, so the two consumers share them rather than each reimplementing.
3801
+ """
3802
+ segments = list(subweeks)
3803
+ if not segments:
3804
+ return {}
3805
+ bounds = segment_capture_bounds(
3806
+ conn, segments, account_key=account_key)
3807
+ by_date: dict = {}
3808
+ for sw in segments:
3809
+ by_date.setdefault(sw.start_date.isoformat(), []).append(sw)
3810
+
3811
+ if rows is None:
3812
+ placeholders = ",".join("?" for _ in by_date)
3813
+ acct_pred = "" if account_key is None else " AND account_key = ?"
3814
+ acct_params: tuple = () if account_key is None else (account_key,)
3815
+ try:
3816
+ fetched = conn.execute(
3817
+ # Held rows are excluded because this is a weekly read.
3818
+ # `_get_latest_row_for_week` and `_floored_week_max` exclude
3819
+ # them for the same stated reason — the three answer which
3820
+ # single row a cycle owns, so they must agree, and two of them
3821
+ # excluding held rows while this one admitted them is what
3822
+ # review finding F1 caught (#769 S11, #824). A store predating
3823
+ # the column raises OperationalError and takes the fallback
3824
+ # below, exactly as it does for `account_key`.
3825
+ "SELECT week_start_date, captured_at_utc, id, weekly_percent "
3826
+ "FROM weekly_usage_snapshots "
3827
+ f"WHERE week_start_date IN ({placeholders}) "
3828
+ f"AND weekly_percent IS NOT NULL"
3829
+ f"{weekly_held_exclusion(conn)}{acct_pred}",
3830
+ tuple(by_date) + acct_params,
3831
+ ).fetchall()
3832
+ except sqlite3.OperationalError:
3833
+ # No table, or a stats.db predating `account_key` (#620 S1).
3834
+ # Withholding is the degradation: retrying unscoped would sum
3835
+ # another account's percentage into an account-filtered total.
3836
+ return {sw.segment_key: None for sw in segments}
3837
+ candidates = []
3838
+ for wsd, captured, row_id, pct in fetched:
3839
+ for sw in by_date.get(str(wsd), ()):
3840
+ candidates.append((sw.segment_key, captured, row_id, pct))
3841
+ else:
3842
+ candidates = list(rows)
3843
+
3844
+ # Bounds are compared as INSTANTS, never as strings. `captured_at_utc` is
3845
+ # written in more than one offset spelling across the tree, and
3846
+ # `'…+00:00' < '…Z'` is TRUE for one instant written two ways — a capture
3847
+ # stamped exactly at a credit would land in the PREDECESSOR, inverting
3848
+ # §1.2's ownership rule. `tests/fixtures/project/credited-week` is written
3849
+ # in the `+00:00` spelling and is where this surfaced.
3850
+ bound_instants = {
3851
+ key: (_segment_instant(since), _segment_instant(before))
3852
+ for key, (since, before) in bounds.items()
3853
+ }
3854
+ as_of_dt = _segment_instant(as_of_utc)
3855
+ best: dict = {}
3856
+ for segment_key, captured, row_id, pct in candidates:
3857
+ if pct is None or not captured:
3858
+ continue
3859
+ captured_dt = _segment_instant(captured)
3860
+ if captured_dt is None:
3861
+ continue
3862
+ since, before = bound_instants.get(segment_key, (None, None))
3863
+ if as_of_dt is not None and captured_dt > as_of_dt:
3864
+ continue
3865
+ if since is not None and captured_dt < since:
3866
+ continue
3867
+ if before is not None and captured_dt >= before:
3868
+ continue
3869
+ rank = (captured_dt, row_id if row_id is not None else -1)
3870
+ current = best.get(segment_key)
3871
+ if current is None or rank > current[0]:
3872
+ best[segment_key] = (rank, float(pct))
3873
+ return {
3874
+ sw.segment_key: (best[sw.segment_key][1]
3875
+ if sw.segment_key in best else None)
3876
+ for sw in segments
3877
+ }
3878
+
3879
+
3355
3880
  def _reset_aware_floor(
3356
3881
  conn: sqlite3.Connection,
3357
3882
  week_start_date: str,
@@ -3370,13 +3895,20 @@ def _reset_aware_floor(
3370
3895
  week); ``None`` is the explicit "all accounts" (merged) read used by the
3371
3896
  analytics floor path, byte-identical to today on a single-account install.
3372
3897
 
3373
- This is the single chokepoint the four MAX-clamp sites consult to floor the
3898
+ This is the single chokepoint the four clamp sites consult to floor the
3374
3899
  current 7d % to the most-recent in-place credit / reset effective moment
3375
- (record-credit M2, issue #209, spec §4a):
3900
+ (record-credit M2, issue #209, spec §4a). The first three take a MAX:
3376
3901
  - statusline `_hwm_clamp` 7d (bin/_cctally_statusline.py)
3377
3902
  - the record-usage write-site monotonic clamp (bin/_cctally_record.py)
3378
3903
  - `_resolve_reset_aware_hwm` (the --from default helper)
3379
- - `project`'s `_load_week_snapshots` per-week MAX (bin/_cctally_project.py)
3904
+
3905
+ The fourth is `project`, and #750 S4 §2.1 changed how it consults this
3906
+ function. It used to reach here through `_load_week_snapshots`'s per-week
3907
+ MAX; that helper (bin/_cctally_project.py) is now a thin wrapper taking no
3908
+ maximum at all, and the floor is resolved per SEGMENT from
3909
+ `segment_capture_bounds` (bin/_cctally_core.py) with that segment's own
3910
+ bounds rather than the week's. The floor still decides which captures a
3911
+ cycle may read; it no longer decides a per-week maximum.
3380
3912
 
3381
3913
  A `week_reset_events` row counts iff its `effective_reset_at_utc` falls in
3382
3914
  `[week_start_at, week_end_at)`; a `weekly_credit_floors` row counts iff its
@@ -3430,9 +3962,20 @@ def _floored_week_max(conn, rows, *, account_key=None):
3430
3962
  suppress the reset-event leg for a later anchored row of the same week.
3431
3963
 
3432
3964
  A NULL ``weekly_percent`` row is skipped. An unparseable ``captured_at_utc``
3433
- under an active floor is RETAINED (epoch unknown), matching
3434
- ``_cctally_project._load_week_snapshots``. All-NULL bounds resolve
3435
- credit-floor-leg-only (the reset leg is inert: unixepoch(NULL) is NULL).
3965
+ under an active floor is RETAINED here (epoch unknown), and that is a
3966
+ DELIBERATE divergence from the two segment-scoped readers of the same
3967
+ table: `_get_latest_row_for_week` compares and orders with `unixepoch()`,
3968
+ which is NULL on an unparseable value, so such a row fails every bound
3969
+ predicate and sorts last under DESC; `latest_usage_by_segment` parses in
3970
+ Python and skips what will not parse. Those two answer "which single row
3971
+ does this cycle own?", where admitting a row whose instant is unknown would
3972
+ attribute it to an arbitrary cycle. This function answers "what is this
3973
+ week's maximum?", where dropping a row would silently lower the answer, and
3974
+ it is the pre-S4 behaviour of every MAX-clamp consumer. The claim that this
3975
+ matches ``_cctally_project._load_week_snapshots`` is no longer true: that
3976
+ helper now delegates to `latest_usage_by_segment` and therefore skips.
3977
+ All-NULL bounds resolve credit-floor-leg-only (the reset leg is inert:
3978
+ unixepoch(NULL) is NULL).
3436
3979
  A week whose every in-scope row is pre-floor is absent from the result.
3437
3980
  """
3438
3981
  # Pass 1: bucket + canonicalize bounds.
@@ -3575,8 +4118,22 @@ def get_latest_usage_for_week(
3575
4118
  as_of_utc: str | None = None,
3576
4119
  *,
3577
4120
  account_key: str | None = None,
4121
+ since_utc: str | None = None,
4122
+ before_utc: str | None = None,
3578
4123
  ) -> sqlite3.Row | None:
4124
+ """The week's latest GENUINELY OBSERVED usage row.
4125
+
4126
+ `exclude_held` (#769 S11, #824): every caller of this helper reads the
4127
+ weekly axis — `report`'s table, the TUI and dashboard trend rows, the
4128
+ milestone cost read and `sync-week` — taking both the percentage and the
4129
+ per-row freshness stamp from the returned row. A `weekly_observation_held`
4130
+ row carries an older row's weekly value under the writing tick's capture
4131
+ instant, so returning one would report a stale percentage as freshly
4132
+ observed. Its five-hour fields are fresh, but nothing reached through here
4133
+ reads them.
4134
+ """
3579
4135
  return _get_latest_row_for_week(
3580
4136
  conn, "weekly_usage_snapshots", week_ref, as_of_utc=as_of_utc,
3581
- account_key=account_key,
4137
+ account_key=account_key, since_utc=since_utc, before_utc=before_utc,
4138
+ exclude_held=True,
3582
4139
  )