cctally 1.108.0 → 1.110.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +147 -0
  2. package/README.md +6 -6
  3. package/bin/_cctally_account.py +4 -0
  4. package/bin/_cctally_alerts.py +6 -2
  5. package/bin/_cctally_cache.py +1652 -166
  6. package/bin/_cctally_config.py +3 -2
  7. package/bin/_cctally_core.py +594 -33
  8. package/bin/_cctally_dashboard.py +1585 -201
  9. package/bin/_cctally_dashboard_conversation.py +494 -48
  10. package/bin/_cctally_dashboard_envelope.py +167 -11
  11. package/bin/_cctally_dashboard_share.py +154 -36
  12. package/bin/_cctally_dashboard_sources.py +2615 -422
  13. package/bin/_cctally_db.py +678 -5
  14. package/bin/_cctally_diagnosis_sources.py +238 -26
  15. package/bin/_cctally_doctor.py +99 -18
  16. package/bin/_cctally_five_hour.py +1295 -40
  17. package/bin/_cctally_forecast.py +124 -28
  18. package/bin/_cctally_journal.py +408 -38
  19. package/bin/_cctally_milestone_history.py +150 -14
  20. package/bin/_cctally_parser.py +5 -0
  21. package/bin/_cctally_percent_breakdown.py +6 -1
  22. package/bin/_cctally_project.py +48 -76
  23. package/bin/_cctally_quota.py +39 -7
  24. package/bin/_cctally_quota_model.py +17 -2
  25. package/bin/_cctally_record.py +1647 -443
  26. package/bin/_cctally_rederive.py +899 -30
  27. package/bin/_cctally_refresh.py +2 -1
  28. package/bin/_cctally_share.py +24 -5
  29. package/bin/_cctally_source_analytics.py +1035 -78
  30. package/bin/_cctally_statusline.py +83 -19
  31. package/bin/_cctally_store.py +61 -4
  32. package/bin/_cctally_transcript.py +27 -2
  33. package/bin/_cctally_tui.py +264 -76
  34. package/bin/_cctally_weekrefs.py +28 -5
  35. package/bin/_lib_blocks.py +127 -20
  36. package/bin/_lib_cache_report.py +20 -5
  37. package/bin/_lib_codex_conversation_query.py +297 -64
  38. package/bin/_lib_codex_hooks.py +102 -0
  39. package/bin/_lib_codex_metadata.py +207 -0
  40. package/bin/_lib_conversation.py +119 -0
  41. package/bin/_lib_conversation_dispatch.py +31 -1
  42. package/bin/_lib_conversation_query.py +315 -51
  43. package/bin/_lib_conversation_retention.py +413 -25
  44. package/bin/_lib_dashboard_sources.py +267 -7
  45. package/bin/_lib_diagnosis.py +10 -0
  46. package/bin/_lib_diff_kernel.py +12 -1
  47. package/bin/_lib_doctor.py +230 -8
  48. package/bin/_lib_ingest_frontier.py +1366 -49
  49. package/bin/_lib_journal.py +19 -2
  50. package/bin/_lib_merge_gate.py +307 -0
  51. package/bin/_lib_pricing.py +588 -29
  52. package/bin/_lib_record.py +130 -12
  53. package/bin/_lib_rederive.py +903 -2
  54. package/bin/_lib_render.py +9 -3
  55. package/bin/_lib_retained_size.py +27 -2
  56. package/bin/_lib_segment_summary.py +66 -8
  57. package/bin/_lib_share_templates.py +28 -18
  58. package/bin/_lib_snapshot_cache.py +115 -1
  59. package/bin/_lib_source_retry.py +320 -0
  60. package/bin/_lib_statusline_candidates.py +400 -36
  61. package/bin/_lib_tick_stats.py +119 -0
  62. package/bin/_lib_view_models.py +256 -76
  63. package/bin/cctally +69 -0
  64. package/dashboard/static/assets/ConversationsView-BB9vUiL6.js +72 -0
  65. package/dashboard/static/assets/{DoctorModal-CWn3U4Wl.js → DoctorModal-DFY2iPDf.js} +1 -1
  66. package/dashboard/static/assets/ModalRoot-DvRKHbQE.js +1 -0
  67. package/dashboard/static/assets/ProjectsDrillPanel-DAv39uSo.js +1 -0
  68. package/dashboard/static/assets/SourceDetailModal-DSAWo2Fp.js +1 -0
  69. package/dashboard/static/assets/{UpdateModal-D3m8GG6V.js → UpdateModal-R_9dOVnh.js} +3 -3
  70. package/dashboard/static/assets/dashboardStream.shared-worker-DbG6Ef0T.js +1 -0
  71. package/dashboard/static/assets/index-COhxe3vU.css +1 -0
  72. package/dashboard/static/assets/index-DdU62sov.js +13 -0
  73. package/dashboard/static/assets/outlineNavigation-BOCLcRa6.js +9 -0
  74. package/dashboard/static/assets/useKeymap-CW7Up0Ay.js +1 -0
  75. package/dashboard/static/dashboard.html +3 -3
  76. package/package.json +4 -1
  77. package/dashboard/static/assets/ConversationsView-BOSaBRtu.js +0 -72
  78. package/dashboard/static/assets/ModalRoot-SR070V6c.js +0 -1
  79. package/dashboard/static/assets/ProjectsDrillPanel-QLI9i5mZ.js +0 -1
  80. package/dashboard/static/assets/SourceDetailModal-pv0wlFxl.js +0 -1
  81. package/dashboard/static/assets/dashboardStream.shared-worker-1XTMV3nr.js +0 -1
  82. package/dashboard/static/assets/index-BHu4mxd8.css +0 -1
  83. package/dashboard/static/assets/index-CIWsbux3.js +0 -13
  84. package/dashboard/static/assets/outlineNavigation-CJvKqmLV.js +0 -9
  85. package/dashboard/static/assets/useKeymap-ffqsS5G0.js +0 -1
@@ -436,7 +436,36 @@ _init_paths_from_env()
436
436
  # is a bump and not a migration is unchanged: the registry is frozen at 13,
437
437
  # and an epoch-current open returns before any schema work, so a handler or an
438
438
  # `add_column_if_missing` would never run on an upgraded install.
439
- STATS_INDEX_EPOCH = 1013
439
+ # 1013 -> 1014 (#769 S2 §3): source-local five-hour credit confirmation state.
440
+ # Adds `five_hour_credit_confirmation_state`, keyed
441
+ # `(account_key, five_hour_window_key, source)`. Five-hour credit detection
442
+ # used to compare an incoming reading against the latest accepted snapshot
443
+ # whatever contributor produced it, so one stale `source=statusline` sample
444
+ # below an `source=api` baseline fabricated a credit with no confirmation from
445
+ # any source (the 2026-09-04 incident, #751). The state carries each
446
+ # contributor's own pre-drop baseline and its pending descent, so a descent
447
+ # observed by one source can only be confirmed by a later, distinct
448
+ # observation from that same source. It is disposable operational state rather
449
+ # than journal truth — the rebuild does not reproduce it and must not count it
450
+ # — and the cold-start consequence of losing it is stated at the helpers in
451
+ # `bin/_cctally_record.py`. The mechanical reason this is a bump and not a
452
+ # migration is unchanged: the registry is frozen at 13, and an epoch-current
453
+ # open returns before any schema work, so a handler or an
454
+ # `add_column_if_missing` would never run on an upgraded install.
455
+ # 1014 -> 1015 (#769 S11, issue #824): weekly_usage_snapshots gains
456
+ # `weekly_observation_held`, which separates a weekly value that was genuinely
457
+ # observed on this tick from one carried forward so a weekly-clamped tick can
458
+ # still persist its five-hour evidence. Before it, a tick whose weekly percent
459
+ # clamped against the stored high-water mark wrote no row at all, so the same
460
+ # tick's genuine five-hour reading was discarded — unclamped and unpersisted.
461
+ # A held row (`weekly_observation_held = 1`) carries the tick's own capture
462
+ # time, source and five-hour fields, while its weekly value and boundary are
463
+ # copied from the latest non-held basis. Weekly readers exclude held rows;
464
+ # five-hour readers include them. The mechanical reason this is a bump and not
465
+ # a migration is unchanged: the registry is frozen at 13, and an epoch-current
466
+ # open returns before any schema work, so a handler or an
467
+ # `add_column_if_missing` would never run on an upgraded install.
468
+ STATS_INDEX_EPOCH = 1015
440
469
  LEGACY_STATS_HEAD = 13
441
470
 
442
471
  #: #496 S1 F1. A NEW branch, for a state that cannot occur before the
@@ -1894,7 +1923,11 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
1894
1923
  import importlib
1895
1924
  _cctally_store = importlib.import_module("_cctally_store")
1896
1925
 
1897
- ensure_dirs()
1926
+ # Alternate targets are private scratch indexes. A rederive preview may
1927
+ # open one while the live data directory has never had a logs/ directory;
1928
+ # creating that directory would violate the preview's zero-write contract.
1929
+ if _target_path is None:
1930
+ ensure_dirs()
1898
1931
  # #453: probe the live index before the maintenance-shared opener. During
1899
1932
  # a large detached rebuild that opener can wait for its bounded 5-second
1900
1933
  # guard; a statusline arriving mid-replay would otherwise pay that delay
@@ -2102,7 +2135,9 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2102
2135
  page_url TEXT,
2103
2136
  source TEXT NOT NULL DEFAULT 'userscript',
2104
2137
  payload_json TEXT NOT NULL,
2105
- account_key TEXT NOT NULL DEFAULT 'unattributed'
2138
+ account_key TEXT NOT NULL DEFAULT 'unattributed',
2139
+ weekly_observation_held INTEGER NOT NULL DEFAULT 0
2140
+ CHECK (weekly_observation_held IN (0, 1))
2106
2141
  )
2107
2142
  """
2108
2143
  )
@@ -2148,6 +2183,19 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2148
2183
  add_column_if_missing(
2149
2184
  conn, "weekly_usage_snapshots", "account_key",
2150
2185
  "TEXT NOT NULL DEFAULT 'unattributed'")
2186
+ # weekly_observation_held (#769 S11, issue #824): the provenance flag
2187
+ # rides the STATS_INDEX_EPOCH 1015 bump, so a fresh rebuild carries it
2188
+ # via the CREATE TABLE above. This backstop keeps an already-1015 index
2189
+ # that predates the column consistent, and covers the interval between
2190
+ # a new binary's first open of an epoch-1014 store and the background
2191
+ # rebuild that resolves the mismatch. Every production writer passes the
2192
+ # flag explicitly; the DEFAULT 0 is the rev-4.1 defensive backstop and
2193
+ # is also the correct value for every row written before the column
2194
+ # existed, because those rows are all genuine weekly observations.
2195
+ add_column_if_missing(
2196
+ conn, "weekly_usage_snapshots", "weekly_observation_held",
2197
+ "INTEGER NOT NULL DEFAULT 0 "
2198
+ "CHECK (weekly_observation_held IN (0, 1))")
2151
2199
  add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_percent", "REAL")
2152
2200
  add_column_if_missing(conn, "weekly_usage_snapshots", "five_hour_resets_at", "TEXT")
2153
2201
  # five_hour_window_key — canonical (10-min-floored epoch) key for
@@ -2283,6 +2331,24 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2283
2331
  # tuple. One index cannot express both, and a single index over the
2284
2332
  # tuple would refuse a second genuine reset that happens to share a
2285
2333
  # boundary pair with the first.
2334
+ # `observed_pre_credit_pct` is a RETAINED NON-INPUT (#761 residual 4,
2335
+ # decision D2). The STORED COLUMN records what the pre-credit reading
2336
+ # was and arms nothing: no read path branches on it, and the clamp floor
2337
+ # that does arm behaviour lives in `weekly_credit_floors`. It is kept,
2338
+ # with its journal payload field, because removing either would break
2339
+ # journal compatibility for no gain, and because it is the only record
2340
+ # of the baseline a historical reset was measured against.
2341
+ #
2342
+ # Do not confuse it with the LIVE FUNCTION ARGUMENT of the same name.
2343
+ # `_fire_in_place_credit(..., observed_pre_credit_pct=...)` is a real
2344
+ # input: it is the band the stale-replica DELETE and its two doomed
2345
+ # captures select on (`_stale_replica_band_sql`). The argument decides
2346
+ # which rows are removed; the column decides nothing.
2347
+ #
2348
+ # This note stays OUTSIDE the SQL. SQLite retains a `CREATE TABLE`
2349
+ # statement verbatim in `sqlite_master.sql`, and `_REBUILD_SCHEMA_
2350
+ # FINGERPRINT` digests that text, so an inline `--` comment here fails
2351
+ # every rebuild with a schema definition mismatch.
2286
2352
  conn.execute(
2287
2353
  """
2288
2354
  CREATE TABLE IF NOT EXISTS week_reset_events (
@@ -2342,6 +2408,39 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2342
2408
  )
2343
2409
  """
2344
2410
  )
2411
+ # #769 S2 §3 epoch 1014: source-local five-hour credit confirmation
2412
+ # state. One row per `(account_key, five_hour_window_key, source)`.
2413
+ #
2414
+ # `baseline_pct` is that contributor's own running maximum inside the
2415
+ # window, held raw — never read back from `weekly_usage_snapshots`,
2416
+ # whose five-hour percent is MAX-clamped at write time and therefore
2417
+ # sits downstream of the defect this table exists to remove.
2418
+ #
2419
+ # The four `pending_*` columns are the armed descent, and they are
2420
+ # nullable together: `pending_low_pct IS NOT NULL` is the armed
2421
+ # predicate. `pending_observation_id` is the arming observation's raw
2422
+ # journal id, so a byte-identical replay of that line cannot confirm
2423
+ # itself; `pending_at_utc` is the arming tick's detection clock, which
2424
+ # becomes the credit's effective instant on confirm.
2425
+ #
2426
+ # Disposable operational state, like `weekly_reset_debounce_state`
2427
+ # above: a rebuild legitimately loses it, and the cold-start
2428
+ # consequence is stated at `_read_five_hour_source_state` in
2429
+ # `bin/_cctally_record.py`.
2430
+ conn.execute(
2431
+ """
2432
+ CREATE TABLE IF NOT EXISTS five_hour_credit_confirmation_state (
2433
+ account_key TEXT NOT NULL,
2434
+ five_hour_window_key INTEGER NOT NULL,
2435
+ source TEXT NOT NULL,
2436
+ baseline_pct REAL NOT NULL,
2437
+ pending_low_pct REAL,
2438
+ pending_at_utc TEXT,
2439
+ pending_observation_id TEXT,
2440
+ PRIMARY KEY (account_key, five_hour_window_key, source)
2441
+ )
2442
+ """
2443
+ )
2345
2444
  _backfill_week_reset_events(conn)
2346
2445
 
2347
2446
  # ── five_hour_reset_events (Anthropic-issued in-place 5h credits) ──
@@ -3202,36 +3301,88 @@ def _get_latest_row_for_week(
3202
3301
  as_of_utc: str | None = None,
3203
3302
  *,
3204
3303
  account_key: str | None = None,
3304
+ since_utc: str | None = None,
3305
+ before_utc: str | None = None,
3306
+ exclude_held: bool = False,
3205
3307
  ) -> sqlite3.Row | None:
3206
3308
  """Latest week row from a weekly snapshot table.
3207
3309
 
3310
+ ``exclude_held`` (#769 S11, #824) drops `weekly_observation_held = 1` rows.
3311
+ It is bound to the CALLER rather than applied unconditionally because this
3312
+ primitive serves two tables: `weekly_usage_snapshots` has the column and
3313
+ `weekly_cost_snapshots` does not, so an unconditional predicate would make
3314
+ every cost lookup raise. `get_latest_usage_for_week` passes it; the cost
3315
+ wrapper in `bin/_cctally_milestones.py` does not.
3316
+
3208
3317
  ``account_key`` (#341): ``None`` = the account-blind merged read (today's
3209
3318
  behavior, byte-stable); a real key scopes to that account's rows — the
3210
3319
  ``--account`` render consumers and the write-path milestone-cost read (P2-1)
3211
- pass it explicitly."""
3212
- acct_pred = "" if account_key is None else " AND account_key = ?"
3213
- acct_params: tuple = () if account_key is None else (account_key,)
3214
- if as_of_utc is None:
3215
- return conn.execute(
3216
- f"""
3217
- SELECT *
3218
- FROM {table_name}
3219
- WHERE week_start_date = ?{acct_pred}
3220
- ORDER BY captured_at_utc DESC, id DESC
3221
- LIMIT 1
3222
- """,
3223
- (week_ref.week_start.isoformat(),) + acct_params,
3224
- ).fetchone()
3320
+ pass it explicitly.
3321
+
3322
+ ``since_utc`` / ``before_utc`` (#750 S4 spec §1.2) bound the read to ONE
3323
+ billing cycle of a credited week: ``since_utc`` is INCLUSIVE and
3324
+ ``before_utc`` is EXCLUSIVE, which is `_aggregate_weekly`'s
3325
+ ``[start_ts, end_ts)`` convention, so a snapshot captured exactly at a
3326
+ credit instant lands in the same segment as an entry stamped there.
3327
+
3328
+ ``as_of_utc`` keeps its inclusive ``<=`` unchanged. It is a different
3329
+ quantity — the caller's global freshness bound — and other callers reach
3330
+ this primitive with it and no segment bounds
3331
+ (`bin/_cctally_sync_week.py`, and `get_latest_cost_for_week` in
3332
+ `bin/_cctally_milestones.py`, which reaches it for the cost table).
3333
+ Redefining it would move them."""
3334
+ preds = ["week_start_date = ?"]
3335
+ params: list = [week_ref.week_start.isoformat()]
3336
+ if as_of_utc is not None:
3337
+ # `unixepoch(...)` here too, for the reason the `since_utc` comment
3338
+ # below gives: a lexical compare disagrees with an instant compare on
3339
+ # two spellings of one moment, and the two consumers of this table
3340
+ # would then read different rows. `captured_at_utc` is stamped at
3341
+ # seconds precision (`now_utc_iso`), so the conversion loses nothing.
3342
+ preds.append("unixepoch(captured_at_utc) <= unixepoch(?)")
3343
+ params.append(as_of_utc)
3344
+ # One consequence, stated because a sibling reader of this table
3345
+ # disagrees with it on purpose: `unixepoch()` is NULL on an
3346
+ # unparseable `captured_at_utc`, so such a row fails this predicate
3347
+ # and sorts LAST under the DESC below. It is excluded here, and
3348
+ # `latest_usage_by_segment` excludes it too, because both answer which
3349
+ # single row a cycle owns and a row whose instant is unknown cannot be
3350
+ # placed in one. `_floored_week_max` RETAINS it, because it answers a
3351
+ # week's maximum and dropping a row would understate that. Neither is
3352
+ # a bug; the reasons are recorded at both sites.
3353
+ if since_utc is not None:
3354
+ # INCLUSIVE lower: the cut instant belongs to the segment that STARTS
3355
+ # there (spec §1.2). Without it a credited tail with no post-cut
3356
+ # capture returns the latest PRE-cut reading — a percent from the
3357
+ # previous cycle — instead of being reported as missing.
3358
+ #
3359
+ # `unixepoch(...)`, never a lexical compare. `captured_at_utc` is
3360
+ # written in more than one offset spelling across the tree, and
3361
+ # `'…+00:00' < '…Z'` is TRUE for one instant written two ways: a
3362
+ # capture stamped exactly at a credit would then fall in the
3363
+ # PREDECESSOR, which is precisely the ownership rule §1.2 fixes. The
3364
+ # same defence `_reset_aware_floor` and `_latest_reset_event_for_end`
3365
+ # already document.
3366
+ preds.append("unixepoch(captured_at_utc) >= unixepoch(?)")
3367
+ params.append(since_utc)
3368
+ if before_utc is not None:
3369
+ # EXCLUSIVE upper: mirrors `_aggregate_weekly`'s [start, end).
3370
+ preds.append("unixepoch(captured_at_utc) < unixepoch(?)")
3371
+ params.append(before_utc)
3372
+ if account_key is not None:
3373
+ preds.append("account_key = ?")
3374
+ params.append(account_key)
3375
+ if exclude_held:
3376
+ preds.append("weekly_observation_held = 0")
3377
+ # Ordered by the PARSED instant, matching how `latest_usage_by_segment`
3378
+ # ranks the same rows. A lexical `captured_at_utc DESC` and an instant
3379
+ # ranking disagree on two spellings of one moment, so the two consumers of
3380
+ # `weekly_usage_snapshots` could pick different rows for one segment. `id`
3381
+ # stays the deterministic tie-breaker, which it already was.
3225
3382
  return conn.execute(
3226
- f"""
3227
- SELECT *
3228
- FROM {table_name}
3229
- WHERE week_start_date = ?
3230
- AND captured_at_utc <= ?{acct_pred}
3231
- ORDER BY captured_at_utc DESC, id DESC
3232
- LIMIT 1
3233
- """,
3234
- (week_ref.week_start.isoformat(), as_of_utc) + acct_params,
3383
+ f"SELECT * FROM {table_name} WHERE " + " AND ".join(preds)
3384
+ + " ORDER BY unixepoch(captured_at_utc) DESC, id DESC LIMIT 1",
3385
+ tuple(params),
3235
3386
  ).fetchone()
3236
3387
 
3237
3388
 
@@ -3352,6 +3503,384 @@ def _ordered_in_place_cuts(
3352
3503
  return ordered
3353
3504
 
3354
3505
 
3506
+ def in_place_cut_instants(
3507
+ conn: sqlite3.Connection, *, account_key: "str | None",
3508
+ ) -> "frozenset[dt.datetime]":
3509
+ """This account's admitted in-place cut instants, UTC-aware.
3510
+
3511
+ #750 S4 spec §1.1. The consumer-side twin of what both appliers decide
3512
+ internally, so `build_weekly_view` and `build_trend_view` cannot drift
3513
+ apart from each other or from the appliers (invariant 1). Five details
3514
+ make it equivalent to them, and each one decides a real case:
3515
+
3516
+ * the in-place TEST is RAW string equality on the unparsed columns,
3517
+ matching `_apply_reset_events_to_subweeks`
3518
+ (bin/_lib_subscription_weeks.py) and `_apply_reset_events_to_weekrefs`
3519
+ (bin/_cctally_weekrefs.py). NOT `unixepoch()`, and NOT
3520
+ `_LEGACY_IN_PLACE_SHAPE` — that predicate is for legacy-row detection
3521
+ and additionally admits the pre-v1.7.2 `(cur_end, cur_end)` shape,
3522
+ which the appliers do not split. A canonicalizing test would bound
3523
+ segments neither applier cut, on rows whose two columns spell one
3524
+ instant in two offsets;
3525
+ * the per-week grouping parses `new_week_end_at`, because that column
3526
+ carries mixed offset spellings, and it is what bounds each cut against
3527
+ its own week's end;
3528
+ * the read is account-scoped, exactly as both appliers scope theirs;
3529
+ * cut instants are deduplicated BY INSTANT and kept unrounded;
3530
+ * a cut at or after its week's end, or at or before a boundary shift
3531
+ that moved that week's start, is rejected through the same
3532
+ `_ordered_in_place_cuts` both appliers call.
3533
+
3534
+ The rows are ordered the way `_latest_reset_event_for_end` orders, and
3535
+ the boundary-shift lower bound is taken with `setdefault`, because that
3536
+ is how both appliers pick the single row that moves a week's start.
3537
+
3538
+ The RESULT is flat rather than keyed by week, because no consumer can
3539
+ reliably name a week's end: `_apply_overlap_clamp_to_subweeks` and a
3540
+ boundary-shift event both move a tail's end away from
3541
+ `new_week_end_at` after the appliers have run. `credited_segment_cuts`
3542
+ is what binds these instants to one week's segments, and it also supplies
3543
+ the remaining lower bound — the week's own start on an unshifted week,
3544
+ which `week_reset_events` does not record.
3545
+
3546
+ An absent `week_reset_events` table yields an empty set: a render-path
3547
+ consumer of an optional table degrades to "this install recorded no
3548
+ credit" rather than raising.
3549
+ """
3550
+ have_table = conn.execute(
3551
+ "SELECT 1 FROM sqlite_master "
3552
+ "WHERE type IN ('table','view') AND name = 'week_reset_events'"
3553
+ ).fetchone()
3554
+ if have_table is None:
3555
+ return frozenset()
3556
+ acct_pred = "" if account_key is None else " WHERE account_key = ?"
3557
+ acct_params: tuple = () if account_key is None else (account_key,)
3558
+ try:
3559
+ rows = conn.execute(
3560
+ "SELECT old_week_end_at, new_week_end_at, effective_reset_at_utc "
3561
+ "FROM week_reset_events" + acct_pred
3562
+ + " ORDER BY unixepoch(effective_reset_at_utc) DESC, id DESC",
3563
+ acct_params,
3564
+ ).fetchall()
3565
+ except sqlite3.OperationalError:
3566
+ # A stats.db predating the `account_key` column (#620 S1): an install
3567
+ # whose migrations have not run, or a dev binary that refused the prod
3568
+ # forward-migration (#142). Withholding is the degradation, exactly as
3569
+ # `_projects_week_grid` withholds on the same predicate — never a
3570
+ # retry without the scope, which would let one account's credit bound
3571
+ # another account's segment.
3572
+ return frozenset()
3573
+ # `post_map` is the boundary shift that moves a week's START; it is the
3574
+ # lower bound every cut of that week must fall after. Only a NON-in-place
3575
+ # row plays that role, which is the same split both appliers make.
3576
+ post_map: dict[dt.datetime, str] = {}
3577
+ cuts_raw: dict[dt.datetime, list[tuple[dt.datetime, str]]] = {}
3578
+ for r in rows:
3579
+ raw_eff = r["effective_reset_at_utc"]
3580
+ try:
3581
+ new_dt = parse_iso_datetime(r["new_week_end_at"], "evt.new_end")
3582
+ except (TypeError, ValueError):
3583
+ continue
3584
+ if r["old_week_end_at"] == raw_eff:
3585
+ try:
3586
+ eff_dt = parse_iso_datetime(raw_eff, "evt.eff")
3587
+ except (TypeError, ValueError):
3588
+ continue
3589
+ cuts_raw.setdefault(new_dt, []).append((eff_dt, raw_eff))
3590
+ else:
3591
+ post_map.setdefault(new_dt, raw_eff)
3592
+ out: set[dt.datetime] = set()
3593
+ for end_dt, cuts in cuts_raw.items():
3594
+ for cut_dt, _raw in _ordered_in_place_cuts(
3595
+ cuts, post_map.get(end_dt), end_dt):
3596
+ out.add(cut_dt.astimezone(dt.timezone.utc))
3597
+ return frozenset(out)
3598
+
3599
+
3600
+ def credited_segment_cuts(
3601
+ segment_starts, cut_instants,
3602
+ ) -> "frozenset[dt.datetime]":
3603
+ """The cut instants that bound ONE week's emitted segments (#750 S4 §1.1).
3604
+
3605
+ ``segment_starts`` are the post-applier start instants of every segment
3606
+ sharing one week identity — `SubWeek.start_date` for the subweeks
3607
+ consumers, `WeekRef.key` for the trend one. ``cut_instants`` is
3608
+ `in_place_cut_instants`. The result is empty for an uncredited week, so
3609
+ ``bool(...)`` is the credited predicate and membership is the per-segment
3610
+ bounded predicate.
3611
+
3612
+ A week's segments are matched to their cuts by their STARTS, never by the
3613
+ week's end. Both appliers emit ``[start, c1)``, each ``[ci, ci+1)`` and
3614
+ ``[cn, end)``, so every cut is verbatim the start of the segment that
3615
+ follows it, while the LAST segment's end is the week end — and two
3616
+ ordinary mechanisms move that end away from the `new_week_end_at` the
3617
+ events table records. `_apply_overlap_clamp_to_subweeks` runs after the
3618
+ reset applier and pulls a tail's end back to the successor's anchor when
3619
+ that anchor drifted earlier, and a boundary-shift event whose
3620
+ `old_week_end_at` is this week's end rewrites the end to the shift
3621
+ instant. Keyed on the week end, the lookup then matched nothing, the week
3622
+ read as uncredited, and BOTH cycles resolved to the week's latest
3623
+ observation — the post-credit reading on the pre-credit cycle, which is
3624
+ #731 / #736 back in full, and silently.
3625
+
3626
+ The EARLIEST start is excluded, which is not a formality: it is the
3627
+ week's own effective start, and both appliers reject a cut at or before
3628
+ it (`_ordered_in_place_cuts`, called with the boundary shift when the
3629
+ week carries one and with the week's own start when it does not). The
3630
+ helper cannot apply that lower bound itself, because `week_reset_events`
3631
+ records no week start; here the segments supply it, so the rule holds on
3632
+ a shifted and an unshifted week alike.
3633
+ """
3634
+ ordered = sorted({s for s in segment_starts if s is not None})
3635
+ return frozenset(s for s in ordered[1:] if s in cut_instants)
3636
+
3637
+
3638
+ def canonical_capture_bound(value_dt: "dt.datetime") -> str:
3639
+ """A segment bound spelled the way ``captured_at_utc`` is stored.
3640
+
3641
+ UTC, whole seconds, ``Z``. SQLite compares those lexicographically, so a
3642
+ bound written in another offset would order wrongly against the stored
3643
+ column. One definition, shared by every consumer that bounds a segment.
3644
+ """
3645
+ return (
3646
+ value_dt.astimezone(dt.timezone.utc)
3647
+ .replace(microsecond=0)
3648
+ .isoformat()
3649
+ .replace("+00:00", "Z")
3650
+ )
3651
+
3652
+
3653
+ def _segment_instant(value: "str | None") -> "dt.datetime | None":
3654
+ if not value:
3655
+ return None
3656
+ try:
3657
+ return parse_iso_datetime(
3658
+ value, "subweek.bound").astimezone(dt.timezone.utc)
3659
+ except (TypeError, ValueError):
3660
+ return None
3661
+
3662
+
3663
+ def segment_capture_bounds(
3664
+ conn: sqlite3.Connection, subweeks, *, account_key: "str | None",
3665
+ ) -> "dict[str, tuple[str | None, str | None]]":
3666
+ """``{SubWeek.segment_key -> (since_utc, before_utc)}`` (#750 S4 §2.1).
3667
+
3668
+ ``since_utc`` is INCLUSIVE and ``before_utc`` EXCLUSIVE, which is
3669
+ `_aggregate_weekly`'s ``[start, end)``, so an observation captured exactly
3670
+ at a credit instant lands in the same cycle as an entry stamped there.
3671
+
3672
+ Only a segment whose END is an in-place cut carries an upper bound. The
3673
+ lower bound is the LATER of two things, and is ``None`` when neither
3674
+ applies:
3675
+
3676
+ * the segment's own start, when its week was credited. That is §1.2's
3677
+ rule, and on the tail it is what makes a cycle with no post-cut
3678
+ observation report as MISSING instead of resolving to the previous
3679
+ cycle's reading;
3680
+ * the latest `week_reset_events` / `weekly_credit_floors` instant lying
3681
+ inside this segment's own interval. That is `_reset_aware_floor`, the
3682
+ chokepoint every other clamp site consults, called per SEGMENT rather
3683
+ than per week. Scoped to the week, its
3684
+ automatic leg discarded pre-credit captures for BOTH segments and so
3685
+ erased the very pre-credit reading it was meant to preserve; scoped to
3686
+ the segment it lands on the tail alone, where it agrees with the first
3687
+ clause. On a boundary-shifted week it still floors at the shift, and a
3688
+ manual `record-credit` floor — written under the same-window model,
3689
+ which re-anchors nothing and creates no segment — still raises the
3690
+ bound of the segment it falls in.
3691
+
3692
+ A segment of a plain uncredited week therefore has NO lower bound, which
3693
+ is what `weekly` does. Imposing one would discard an observation captured
3694
+ a little BEFORE its week's anchor — the mirror of the post-boundary
3695
+ capture jitter the weekly surfaces already tolerate, and a real shape in
3696
+ the fixtures.
3697
+ """
3698
+ segments = list(subweeks)
3699
+ if not segments:
3700
+ return {}
3701
+ cut_instants = in_place_cut_instants(conn, account_key=account_key)
3702
+ by_date: dict = {}
3703
+ for sw in segments:
3704
+ by_date.setdefault(sw.start_date.isoformat(), []).append(sw)
3705
+
3706
+ out: dict = {}
3707
+ for _date_key, group in by_date.items():
3708
+ # Matched by the segments' own STARTS — a cut is verbatim the start
3709
+ # of the segment that follows it, while a tail's END can be moved
3710
+ # after the appliers run by the overlap clamp or a boundary shift.
3711
+ week_cuts = credited_segment_cuts(
3712
+ (_segment_instant(sw.start_ts) for sw in group), cut_instants)
3713
+ credited = bool(week_cuts)
3714
+ for sw in group:
3715
+ start_dt = _segment_instant(sw.start_ts)
3716
+ end_dt = _segment_instant(sw.end_ts)
3717
+ lower = start_dt if (credited and start_dt is not None) else None
3718
+ # The floor comes from the SAME chokepoint every other clamp site
3719
+ # consults, called with this SEGMENT's bounds rather than the
3720
+ # week's. Scoped to the week, its reset leg discarded pre-credit
3721
+ # captures for both cycles; scoped to the segment it lands on the
3722
+ # tail alone. The returned instant is re-checked against the
3723
+ # segment interval because the manual `weekly_credit_floors` leg
3724
+ # is keyed on the week's date and carries no interval bound of
3725
+ # its own.
3726
+ try:
3727
+ floor_dt = _segment_instant(_reset_aware_floor(
3728
+ conn, sw.start_date.isoformat(), sw.start_ts, sw.end_ts,
3729
+ account_key=account_key,
3730
+ ))
3731
+ except sqlite3.OperationalError:
3732
+ # A store with neither floor table: an install whose
3733
+ # migrations have not run, or a unit fixture holding only the
3734
+ # snapshot table. The leg is absent, not wrong.
3735
+ floor_dt = None
3736
+ if floor_dt is not None:
3737
+ inside = (
3738
+ (start_dt is None or floor_dt >= start_dt)
3739
+ and (end_dt is None or floor_dt < end_dt)
3740
+ )
3741
+ if inside and (lower is None or floor_dt > lower):
3742
+ lower = floor_dt
3743
+ before = (
3744
+ canonical_capture_bound(end_dt)
3745
+ if (end_dt is not None and end_dt in week_cuts) else None
3746
+ )
3747
+ out[sw.segment_key] = (
3748
+ canonical_capture_bound(lower) if lower is not None else None,
3749
+ before,
3750
+ )
3751
+ return out
3752
+
3753
+
3754
+ def weekly_held_exclusion(conn, *, prefix: str = " AND ") -> str:
3755
+ """``AND weekly_observation_held = 0``, or empty on a store without it.
3756
+
3757
+ Omitting the predicate where the column is absent is not a compromise: the
3758
+ column arrived with epoch 1015, and a store that does not carry it cannot
3759
+ contain a held row, so the filtered and unfiltered populations are
3760
+ identical. Production never takes that branch — `open_db` raises
3761
+ `StatsEpochRebuildDeferred` on an epoch-mismatched store before any read,
3762
+ and a legacy store gains the column at the schema apply — but several
3763
+ hand-built fixture stores stop short of the current schema, and the readers
3764
+ that use this already tolerate an older shape by design.
3765
+ """
3766
+ try:
3767
+ cols = {str(row[1]) for row in
3768
+ conn.execute("PRAGMA table_info(weekly_usage_snapshots)")}
3769
+ except sqlite3.Error:
3770
+ return ""
3771
+ if "weekly_observation_held" not in cols:
3772
+ return ""
3773
+ return f"{prefix}weekly_observation_held = 0"
3774
+
3775
+
3776
+ def latest_usage_by_segment(
3777
+ conn: sqlite3.Connection, subweeks, *,
3778
+ account_key: "str | None" = None,
3779
+ as_of_utc: "str | None" = None,
3780
+ rows=None,
3781
+ ) -> "dict[str, float | None]":
3782
+ """``{SubWeek.segment_key -> the latest weekly_percent in that cycle}``.
3783
+
3784
+ #750 S4 §2.1. One value per SEGMENT identity, never per calendar date:
3785
+ `_load_week_snapshots` returned ``{week_start_utc -> max(weekly_percent)}``
3786
+ keyed on `weekly_usage_snapshots.week_start_at`, so on a credited week the
3787
+ post-credit segment matched nothing and the pre-credit one received the
3788
+ post-credit reading (#731). A segment with no observation inside its own
3789
+ interval maps to ``None`` rather than borrowing a neighbour's.
3790
+
3791
+ LATEST, not MAX. "weekly_percent is monotonic within a week" is true of a
3792
+ billing CYCLE and false of a credited week, which is why the dashboard
3793
+ projects query already reads the latest row. Taking the maximum would lock
3794
+ a credited week's attribution to the pre-credit high-water mark forever.
3795
+ Inside one cycle the two agree.
3796
+
3797
+ It takes the CALLER's connection: `_load_week_snapshots` opened its own
3798
+ while the dashboard already holds a transaction, and a reducer that opened
3799
+ its own would give the dashboard a second, possibly inconsistent read.
3800
+
3801
+ ``rows`` lets a caller that has already fetched and resolved its own rows
3802
+ hand them over as ``(segment_key, captured_at_utc, row_id,
3803
+ weekly_percent)`` tuples; the bound and latest-wins decisions still happen
3804
+ here, so the two consumers share them rather than each reimplementing.
3805
+ """
3806
+ segments = list(subweeks)
3807
+ if not segments:
3808
+ return {}
3809
+ bounds = segment_capture_bounds(
3810
+ conn, segments, account_key=account_key)
3811
+ by_date: dict = {}
3812
+ for sw in segments:
3813
+ by_date.setdefault(sw.start_date.isoformat(), []).append(sw)
3814
+
3815
+ if rows is None:
3816
+ placeholders = ",".join("?" for _ in by_date)
3817
+ acct_pred = "" if account_key is None else " AND account_key = ?"
3818
+ acct_params: tuple = () if account_key is None else (account_key,)
3819
+ try:
3820
+ fetched = conn.execute(
3821
+ # Held rows are excluded because this is a weekly read.
3822
+ # `_get_latest_row_for_week` and `_floored_week_max` exclude
3823
+ # them for the same stated reason — the three answer which
3824
+ # single row a cycle owns, so they must agree, and two of them
3825
+ # excluding held rows while this one admitted them is what
3826
+ # review finding F1 caught (#769 S11, #824). A store predating
3827
+ # the column raises OperationalError and takes the fallback
3828
+ # below, exactly as it does for `account_key`.
3829
+ "SELECT week_start_date, captured_at_utc, id, weekly_percent "
3830
+ "FROM weekly_usage_snapshots "
3831
+ f"WHERE week_start_date IN ({placeholders}) "
3832
+ f"AND weekly_percent IS NOT NULL"
3833
+ f"{weekly_held_exclusion(conn)}{acct_pred}",
3834
+ tuple(by_date) + acct_params,
3835
+ ).fetchall()
3836
+ except sqlite3.OperationalError:
3837
+ # No table, or a stats.db predating `account_key` (#620 S1).
3838
+ # Withholding is the degradation: retrying unscoped would sum
3839
+ # another account's percentage into an account-filtered total.
3840
+ return {sw.segment_key: None for sw in segments}
3841
+ candidates = []
3842
+ for wsd, captured, row_id, pct in fetched:
3843
+ for sw in by_date.get(str(wsd), ()):
3844
+ candidates.append((sw.segment_key, captured, row_id, pct))
3845
+ else:
3846
+ candidates = list(rows)
3847
+
3848
+ # Bounds are compared as INSTANTS, never as strings. `captured_at_utc` is
3849
+ # written in more than one offset spelling across the tree, and
3850
+ # `'…+00:00' < '…Z'` is TRUE for one instant written two ways — a capture
3851
+ # stamped exactly at a credit would land in the PREDECESSOR, inverting
3852
+ # §1.2's ownership rule. `tests/fixtures/project/credited-week` is written
3853
+ # in the `+00:00` spelling and is where this surfaced.
3854
+ bound_instants = {
3855
+ key: (_segment_instant(since), _segment_instant(before))
3856
+ for key, (since, before) in bounds.items()
3857
+ }
3858
+ as_of_dt = _segment_instant(as_of_utc)
3859
+ best: dict = {}
3860
+ for segment_key, captured, row_id, pct in candidates:
3861
+ if pct is None or not captured:
3862
+ continue
3863
+ captured_dt = _segment_instant(captured)
3864
+ if captured_dt is None:
3865
+ continue
3866
+ since, before = bound_instants.get(segment_key, (None, None))
3867
+ if as_of_dt is not None and captured_dt > as_of_dt:
3868
+ continue
3869
+ if since is not None and captured_dt < since:
3870
+ continue
3871
+ if before is not None and captured_dt >= before:
3872
+ continue
3873
+ rank = (captured_dt, row_id if row_id is not None else -1)
3874
+ current = best.get(segment_key)
3875
+ if current is None or rank > current[0]:
3876
+ best[segment_key] = (rank, float(pct))
3877
+ return {
3878
+ sw.segment_key: (best[sw.segment_key][1]
3879
+ if sw.segment_key in best else None)
3880
+ for sw in segments
3881
+ }
3882
+
3883
+
3355
3884
  def _reset_aware_floor(
3356
3885
  conn: sqlite3.Connection,
3357
3886
  week_start_date: str,
@@ -3370,13 +3899,20 @@ def _reset_aware_floor(
3370
3899
  week); ``None`` is the explicit "all accounts" (merged) read used by the
3371
3900
  analytics floor path, byte-identical to today on a single-account install.
3372
3901
 
3373
- This is the single chokepoint the four MAX-clamp sites consult to floor the
3902
+ This is the single chokepoint the four clamp sites consult to floor the
3374
3903
  current 7d % to the most-recent in-place credit / reset effective moment
3375
- (record-credit M2, issue #209, spec §4a):
3904
+ (record-credit M2, issue #209, spec §4a). The first three take a MAX:
3376
3905
  - statusline `_hwm_clamp` 7d (bin/_cctally_statusline.py)
3377
3906
  - the record-usage write-site monotonic clamp (bin/_cctally_record.py)
3378
3907
  - `_resolve_reset_aware_hwm` (the --from default helper)
3379
- - `project`'s `_load_week_snapshots` per-week MAX (bin/_cctally_project.py)
3908
+
3909
+ The fourth is `project`, and #750 S4 §2.1 changed how it consults this
3910
+ function. It used to reach here through `_load_week_snapshots`'s per-week
3911
+ MAX; that helper (bin/_cctally_project.py) is now a thin wrapper taking no
3912
+ maximum at all, and the floor is resolved per SEGMENT from
3913
+ `segment_capture_bounds` (bin/_cctally_core.py) with that segment's own
3914
+ bounds rather than the week's. The floor still decides which captures a
3915
+ cycle may read; it no longer decides a per-week maximum.
3380
3916
 
3381
3917
  A `week_reset_events` row counts iff its `effective_reset_at_utc` falls in
3382
3918
  `[week_start_at, week_end_at)`; a `weekly_credit_floors` row counts iff its
@@ -3430,9 +3966,20 @@ def _floored_week_max(conn, rows, *, account_key=None):
3430
3966
  suppress the reset-event leg for a later anchored row of the same week.
3431
3967
 
3432
3968
  A NULL ``weekly_percent`` row is skipped. An unparseable ``captured_at_utc``
3433
- under an active floor is RETAINED (epoch unknown), matching
3434
- ``_cctally_project._load_week_snapshots``. All-NULL bounds resolve
3435
- credit-floor-leg-only (the reset leg is inert: unixepoch(NULL) is NULL).
3969
+ under an active floor is RETAINED here (epoch unknown), and that is a
3970
+ DELIBERATE divergence from the two segment-scoped readers of the same
3971
+ table: `_get_latest_row_for_week` compares and orders with `unixepoch()`,
3972
+ which is NULL on an unparseable value, so such a row fails every bound
3973
+ predicate and sorts last under DESC; `latest_usage_by_segment` parses in
3974
+ Python and skips what will not parse. Those two answer "which single row
3975
+ does this cycle own?", where admitting a row whose instant is unknown would
3976
+ attribute it to an arbitrary cycle. This function answers "what is this
3977
+ week's maximum?", where dropping a row would silently lower the answer, and
3978
+ it is the pre-S4 behaviour of every MAX-clamp consumer. The claim that this
3979
+ matches ``_cctally_project._load_week_snapshots`` is no longer true: that
3980
+ helper now delegates to `latest_usage_by_segment` and therefore skips.
3981
+ All-NULL bounds resolve credit-floor-leg-only (the reset leg is inert:
3982
+ unixepoch(NULL) is NULL).
3436
3983
  A week whose every in-scope row is pre-floor is absent from the result.
3437
3984
  """
3438
3985
  # Pass 1: bucket + canonicalize bounds.
@@ -3575,8 +4122,22 @@ def get_latest_usage_for_week(
3575
4122
  as_of_utc: str | None = None,
3576
4123
  *,
3577
4124
  account_key: str | None = None,
4125
+ since_utc: str | None = None,
4126
+ before_utc: str | None = None,
3578
4127
  ) -> sqlite3.Row | None:
4128
+ """The week's latest GENUINELY OBSERVED usage row.
4129
+
4130
+ `exclude_held` (#769 S11, #824): every caller of this helper reads the
4131
+ weekly axis — `report`'s table, the TUI and dashboard trend rows, the
4132
+ milestone cost read and `sync-week` — taking both the percentage and the
4133
+ per-row freshness stamp from the returned row. A `weekly_observation_held`
4134
+ row carries an older row's weekly value under the writing tick's capture
4135
+ instant, so returning one would report a stale percentage as freshly
4136
+ observed. Its five-hour fields are fresh, but nothing reached through here
4137
+ reads them.
4138
+ """
3579
4139
  return _get_latest_row_for_week(
3580
4140
  conn, "weekly_usage_snapshots", week_ref, as_of_utc=as_of_utc,
3581
- account_key=account_key,
4141
+ account_key=account_key, since_utc=since_utc, before_utc=before_utc,
4142
+ exclude_held=True,
3582
4143
  )