cctally 1.113.0 → 1.115.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +47 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_account.py +32 -8
  4. package/bin/_cctally_cache.py +6035 -722
  5. package/bin/_cctally_completion.py +1287 -0
  6. package/bin/_cctally_config.py +37 -0
  7. package/bin/_cctally_core.py +400 -2
  8. package/bin/_cctally_dashboard.py +98 -20
  9. package/bin/_cctally_dashboard_conversation.py +292 -23
  10. package/bin/_cctally_dashboard_envelope.py +16 -3
  11. package/bin/_cctally_dashboard_perf.py +128 -9
  12. package/bin/_cctally_dashboard_sources.py +285 -213
  13. package/bin/_cctally_db.py +1177 -23
  14. package/bin/_cctally_diagnosis_sources.py +22 -4
  15. package/bin/_cctally_doctor.py +572 -45
  16. package/bin/_cctally_five_hour.py +84 -33
  17. package/bin/_cctally_forecast.py +108 -15
  18. package/bin/_cctally_journal.py +3950 -116
  19. package/bin/_cctally_journal_repair.py +101 -12
  20. package/bin/_cctally_milestone_history.py +82 -9
  21. package/bin/_cctally_parser.py +42 -0
  22. package/bin/_cctally_percent_breakdown.py +3 -2
  23. package/bin/_cctally_project.py +8 -2
  24. package/bin/_cctally_quarantine.py +652 -0
  25. package/bin/_cctally_quota.py +227 -37
  26. package/bin/_cctally_quota_model.py +360 -83
  27. package/bin/_cctally_record.py +691 -149
  28. package/bin/_cctally_rederive.py +320 -41
  29. package/bin/_cctally_refresh.py +84 -13
  30. package/bin/_cctally_source_analytics.py +83 -17
  31. package/bin/_cctally_statusline.py +6154 -517
  32. package/bin/_cctally_store.py +367 -53
  33. package/bin/_cctally_transcript.py +8 -7
  34. package/bin/_cctally_tui.py +61 -56
  35. package/bin/_cctally_weekrefs.py +128 -21
  36. package/bin/_lib_account_binding.py +152 -0
  37. package/bin/_lib_account_protocol.py +386 -0
  38. package/bin/_lib_aggregators.py +4 -0
  39. package/bin/_lib_cache_coverage.py +13 -1
  40. package/bin/_lib_cache_report.py +35 -6
  41. package/bin/_lib_cache_writer_lock.py +19 -3
  42. package/bin/_lib_clone_copy.py +142 -0
  43. package/bin/_lib_codex_conversation.py +32 -0
  44. package/bin/_lib_codex_conversation_query.py +297 -49
  45. package/bin/_lib_codex_conversation_watch.py +59 -14
  46. package/bin/_lib_codex_drain_admission.py +214 -0
  47. package/bin/_lib_codex_metadata.py +18 -1
  48. package/bin/_lib_codex_publication_barrier.py +277 -0
  49. package/bin/_lib_codex_rollout_container.py +502 -0
  50. package/bin/_lib_conversation_dispatch.py +3 -0
  51. package/bin/_lib_conversation_query.py +106 -59
  52. package/bin/_lib_conversation_retention.py +1605 -640
  53. package/bin/_lib_cost_provenance.py +53 -2
  54. package/bin/_lib_detector_access_registry.py +3817 -0
  55. package/bin/_lib_doctor.py +714 -69
  56. package/bin/_lib_five_hour.py +126 -0
  57. package/bin/_lib_fmt.py +11 -0
  58. package/bin/_lib_ingest_frontier.py +414 -20
  59. package/bin/_lib_journal.py +141 -10
  60. package/bin/_lib_jsonl.py +5 -4
  61. package/bin/_lib_legacy_spool.py +126 -0
  62. package/bin/_lib_legacy_writer_evidence.py +136 -0
  63. package/bin/_lib_merge_gate.py +231 -86
  64. package/bin/_lib_meter_rate_change.py +10 -0
  65. package/bin/_lib_outcome_quarantine.py +690 -0
  66. package/bin/_lib_perf.py +27 -1
  67. package/bin/_lib_pricing.py +199 -9
  68. package/bin/_lib_pricing_check.py +20 -4
  69. package/bin/_lib_quarantine.py +154 -0
  70. package/bin/_lib_quota_calibration.py +53 -8
  71. package/bin/_lib_quota_copy.py +2 -1
  72. package/bin/_lib_quota_model.py +288 -46
  73. package/bin/_lib_reclaim_planner.py +792 -0
  74. package/bin/_lib_rederive.py +319 -82
  75. package/bin/_lib_segment_summary.py +35 -2
  76. package/bin/_lib_snapshot_cache.py +21 -0
  77. package/bin/_lib_sqlite_close.py +81 -0
  78. package/bin/_lib_statusline_candidates.py +292 -23
  79. package/bin/_lib_subscription_weeks.py +100 -17
  80. package/bin/_lib_tick_stats.py +217 -17
  81. package/bin/_lib_wal_checkpoint.py +1077 -0
  82. package/bin/_lib_write_budget.py +483 -0
  83. package/bin/_lib_write_io.py +703 -0
  84. package/bin/cctally +31 -1
  85. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  86. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  87. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  88. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  89. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  90. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  91. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  92. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  93. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  94. package/dashboard/static/dashboard.html +2 -2
  95. package/package.json +19 -1
  96. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  97. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -529,6 +529,43 @@ def resolve_retention_days(config: dict) -> int:
529
529
  return default
530
530
 
531
531
 
532
+ #: The test-only switch (#965). Honored only at exactly "1", only when no
533
+ #: retention setting exists, and only for code running from a git checkout.
534
+ CONVERSATION_RETENTION_TEST_SWITCH = "CCTALLY_TEST_CONVERSATION_RETENTION_DEFAULT_OFF"
535
+
536
+
537
+ def _conversation_retention_setting_absent(config) -> bool:
538
+ """True when no ``conversation.retention_days`` setting exists (#965 5.2).
539
+
540
+ Absent: no ``conversation`` key, or a ``conversation`` dictionary without
541
+ ``retention_days``. A present key of any value (``null`` and malformed
542
+ values included), a non-dictionary ``conversation`` and a non-dictionary
543
+ config are not absent: they keep ``resolve_retention_days``'s meaning."""
544
+ if not isinstance(config, dict):
545
+ return False
546
+ if "conversation" not in config:
547
+ return True
548
+ block = config["conversation"]
549
+ return isinstance(block, dict) and "retention_days" not in block
550
+
551
+
552
+ def resolve_prune_retention_days(config: dict) -> int:
553
+ """Operational transcript-retention window of the three prune paths (#965).
554
+
555
+ ``resolve_retention_days`` stays the reporting resolver (``config get``,
556
+ the settings manifest, docs). This one differs in exactly one case: with
557
+ no retention setting, the test switch at exactly ``"1"`` and a ``.git``
558
+ entry at the running code's root, it returns ``0`` (keep forever). The
559
+ switch and the marker are read on every call; nothing is cached, written
560
+ or logged. docs/fixtures-gotchas.md, "Conversation retention is off in
561
+ test environments (#965)"."""
562
+ if (_conversation_retention_setting_absent(config)
563
+ and os.environ.get(CONVERSATION_RETENTION_TEST_SWITCH) == "1"
564
+ and _cctally_core._has_git_checkout_marker()):
565
+ return 0
566
+ return resolve_retention_days(config)
567
+
568
+
532
569
  def _config_codex_leaf_value(config: dict, key: str) -> object:
533
570
  """Read one nested Codex budget leaf with its public default.
534
571
 
@@ -76,6 +76,8 @@ def _init_paths_from_env() -> None:
76
76
  global STATUSLINE_CANDIDATE_DIR, STATUSLINE_SELECTED_PATH
77
77
  global STATUSLINE_TRANSPORT_MARKER_PATH
78
78
  global STATUSLINE_AUTHORITATIVE_7D_PATH, STATUSLINE_AUTHORITATIVE_5H_PATH
79
+ global STATUSLINE_ACCOUNT_BOUND_DIR, LEGACY_SPOOL_BASELINE_PATH
80
+ global ACCOUNT_BOUND_CUTOVER_CACHE_PATH, LEGACY_WRITER_EVIDENCE_PATH
79
81
  global OAUTH_BACKOFF_MARKER_PATH, OAUTH_BACKOFF_COUNT_PATH
80
82
  global UPDATE_STATE_PATH, UPDATE_SUPPRESS_PATH
81
83
  global UPDATE_LOCK_PATH, UPDATE_LOG_PATH, UPDATE_LOG_ROTATED_PATH
@@ -184,6 +186,19 @@ def _init_paths_from_env() -> None:
184
186
  STATUSLINE_TRANSPORT_MARKER_PATH = APP_DIR / "statusline-transport.last"
185
187
  STATUSLINE_AUTHORITATIVE_7D_PATH = APP_DIR / "statusline-authoritative-7d.json"
186
188
  STATUSLINE_AUTHORITATIVE_5H_PATH = APP_DIR / "statusline-authoritative-5h.json"
189
+ # #825/#832 account-bound protocol (spec section 7). Every current
190
+ # coordination file lives under its own sibling root; the legacy paths
191
+ # above are never read as authority after the cutover, only compared with
192
+ # the journal-anchored legacy spool baseline. The coordinate cache only
193
+ # accelerates locating the cutover op and never confers authority; the
194
+ # legacy-writer evidence sidecar is diagnostic only (spec section 8).
195
+ STATUSLINE_ACCOUNT_BOUND_DIR = APP_DIR / "statusline-account-bound-v1"
196
+ LEGACY_SPOOL_BASELINE_PATH = (
197
+ STATUSLINE_ACCOUNT_BOUND_DIR / "legacy-spool-baseline.json")
198
+ ACCOUNT_BOUND_CUTOVER_CACHE_PATH = (
199
+ STATUSLINE_ACCOUNT_BOUND_DIR / "cutover-coordinate.json")
200
+ LEGACY_WRITER_EVIDENCE_PATH = (
201
+ STATUSLINE_ACCOUNT_BOUND_DIR / "legacy-writer-evidence.json")
187
202
  OAUTH_BACKOFF_MARKER_PATH = APP_DIR / "oauth-backoff.until"
188
203
  # Consecutive-429 counter (text int) driving the headerless exponential
189
204
  # backoff (base * 2**count). Separate from the deadline marker so the
@@ -261,6 +276,18 @@ def _repo_root() -> pathlib.Path:
261
276
  return pathlib.Path(__file__).resolve().parent.parent
262
277
 
263
278
 
279
+ def _has_git_checkout_marker() -> bool:
280
+ """True iff the running code has a ``.git`` entry at its root (#965 5.1).
281
+
282
+ A main checkout has a ``.git`` directory and a linked worktree a ``.git``
283
+ file; both count. Eligibility follows the code's resolved location
284
+ (``_repo_root()``), never the current directory, and is evaluated on every
285
+ call. No suppressor (``CCTALLY_DISABLE_DEV_AUTODETECT``,
286
+ ``CCTALLY_ALLOW_PROD_MIGRATION``) affects it. npm and Homebrew installs
287
+ ship without ``.git``, so they never read True."""
288
+ return (_repo_root() / ".git").exists()
289
+
290
+
264
291
  def _is_dev_checkout() -> bool:
265
292
  """True iff running from a git checkout (a ``.git`` entry at the repo
266
293
  root — a directory for a main checkout, a file for a worktree) AND the
@@ -272,7 +299,7 @@ def _is_dev_checkout() -> bool:
272
299
  install copies ship without ``.git`` so they never read True."""
273
300
  if _truthy_env("CCTALLY_DISABLE_DEV_AUTODETECT"):
274
301
  return False
275
- return (_repo_root() / ".git").exists()
302
+ return _has_git_checkout_marker()
276
303
 
277
304
 
278
305
  def is_preview_channel() -> bool:
@@ -473,9 +500,33 @@ _init_paths_from_env()
473
500
  # tick must carry that earlier evidence into the journal, even after this
474
501
  # binary learns a new model card. A pre-existing open projection that was
475
502
  # rebuilt without the marker remains unproven until a new tick recomputes it.
476
- STATS_INDEX_EPOCH = 1016
503
+ # 1016 -> 1017 (#901 §5.3, T3): read indexes only. Every stats digest relation
504
+ # (`_lib_dashboard_sources._CODEX_STATS_DIGEST_RELATIONS` and
505
+ # `_CLAUDE_STATS_DIGEST_RELATIONS`) sorted its whole relation through a temp
506
+ # b-tree on every dashboard build, and `_fetch_current_week_snapshots` grouped
507
+ # every boundary-aware and legacy-date snapshot the same way; each now streams
508
+ # through an index in its own order (`_STATS_READ_INDEX_DDL`). No table or
509
+ # column changes and no digest byte moves. The same epoch carries Amendment 1's
510
+ # history-read indexes (items 2-7 and 9, `c901-design-2`, with item 2's second
511
+ # single-root index from Amendment 1c, `c901-design-3`): every full dashboard
512
+ # build enumerated weekly blocks, milestone weeks, root keys, group pairs,
513
+ # reset events and subscription anchors through temp b-trees. The mechanical
514
+ # reason this is a bump and not a migration is unchanged: the registry is
515
+ # frozen at 13, and an epoch-current open returns before any schema work, so a
516
+ # schema helper alone would never run on an upgraded install.
517
+ # 1017 -> 1018 (#825/#832 spec section 6): the same journal now has a different
518
+ # authoritative interpretation — the account-bound admission predicate excludes
519
+ # unstamped post-cutover records before any derivation — so an index
520
+ # materialized under the old interpretation is not current even though its
521
+ # schema is usable. The epoch also carries the disposable
522
+ # `journal_admission_summary` table, whose presence is the protocol marker
523
+ # `_cctally_store.validated_read_snapshot` checks inside each decision
524
+ # scope's own snapshot. An epoch transition, not a numbered migration: the
525
+ # registry stays frozen at 13.
526
+ STATS_INDEX_EPOCH = 1018
477
527
  LEGACY_STATS_HEAD = 13
478
528
 
529
+
479
530
  #: #496 S1 F1. A NEW branch, for a state that cannot occur before the
480
531
  #: publication transaction exists: a replacement index was published and then
481
532
  #: failed validation on a fresh connection. The existing corrupt-stats text
@@ -1861,6 +1912,207 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
1861
1912
  "next_evaluation_by_root_json", "TEXT NOT NULL DEFAULT '{}'")
1862
1913
 
1863
1914
 
1915
+ #: #901 §5.3 (T3), epoch 1017: (table, columns the index reads, DDL). Each
1916
+ #: index's leading columns are the statement's equality filter (when it has
1917
+ #: one) followed by its ORDER BY / GROUP BY list, so the statement streams off
1918
+ #: the index with no sorter; the digest indexes are covering, because every
1919
+ #: digest relation selects exactly its ordering columns. SQLite stores each
1920
+ #: statement verbatim in `sqlite_schema`, which `_REBUILD_SCHEMA_FINGERPRINT`
1921
+ #: hashes: change the text and the fingerprint must move with it.
1922
+ _STATS_READ_INDEX_DDL: "tuple[tuple[str, tuple[str, ...], str], ...]" = (
1923
+ ("quota_projection_state", ("source_root_key", "physical_signature"),
1924
+ "CREATE INDEX IF NOT EXISTS idx_quota_projection_state_digest "
1925
+ "ON quota_projection_state(source_root_key, physical_signature)"),
1926
+ ("quota_window_blocks",
1927
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1928
+ "window_minutes", "limit_id", "limit_name", "resets_at_utc",
1929
+ "nominal_start_at_utc", "first_observed_at_utc", "last_observed_at_utc",
1930
+ "first_percent", "current_percent", "orphaned_at"),
1931
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_digest "
1932
+ "ON quota_window_blocks(source, source_root_key, logical_limit_key, "
1933
+ "observed_slot, window_minutes, limit_id, limit_name, resets_at_utc, "
1934
+ "nominal_start_at_utc, first_observed_at_utc, last_observed_at_utc, "
1935
+ "first_percent, current_percent, orphaned_at) WHERE source='codex'"),
1936
+ ("quota_percent_milestones",
1937
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1938
+ "window_minutes", "resets_at_utc", "percent_threshold",
1939
+ "captured_at_utc", "high_water_percent", "orphaned_at"),
1940
+ "CREATE INDEX IF NOT EXISTS idx_quota_milestones_digest "
1941
+ "ON quota_percent_milestones(source, source_root_key, logical_limit_key, "
1942
+ "observed_slot, window_minutes, resets_at_utc, percent_threshold, "
1943
+ "captured_at_utc, high_water_percent, orphaned_at) WHERE source='codex'"),
1944
+ ("quota_threshold_events",
1945
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1946
+ "window_minutes", "resets_at_utc", "threshold", "qualifying_kind",
1947
+ "qualifying_percent", "projected_percent", "severity", "created_at_utc",
1948
+ "disposition", "alerted_at", "suppressed_at", "orphaned_at"),
1949
+ "CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_digest "
1950
+ "ON quota_threshold_events(source, source_root_key, logical_limit_key, "
1951
+ "observed_slot, window_minutes, resets_at_utc, threshold, "
1952
+ "qualifying_kind, qualifying_percent, projected_percent, severity, "
1953
+ "created_at_utc, disposition, alerted_at, suppressed_at, orphaned_at) "
1954
+ "WHERE source='codex'"),
1955
+ ("budget_milestones",
1956
+ ("vendor", "period_start_at", "period", "threshold", "budget_usd",
1957
+ "spent_usd", "consumption_pct", "crossed_at_utc", "alerted_at"),
1958
+ "CREATE INDEX IF NOT EXISTS idx_budget_milestones_codex_digest "
1959
+ "ON budget_milestones(vendor, period_start_at, period, threshold, "
1960
+ "budget_usd, spent_usd, consumption_pct, crossed_at_utc, alerted_at) "
1961
+ "WHERE vendor='codex'"),
1962
+ ("projected_milestones",
1963
+ ("metric", "week_start_at", "period", "threshold", "projected_value",
1964
+ "denominator", "crossed_at_utc", "alerted_at"),
1965
+ "CREATE INDEX IF NOT EXISTS idx_projected_milestones_codex_digest "
1966
+ "ON projected_milestones(metric, week_start_at, period, threshold, "
1967
+ "projected_value, denominator, crossed_at_utc, alerted_at) "
1968
+ "WHERE metric='codex_budget_usd'"),
1969
+ ("percent_milestones",
1970
+ ("week_start_date", "percent_threshold", "reset_event_id", "account_key",
1971
+ "captured_at_utc", "cumulative_cost_usd", "alerted_at"),
1972
+ "CREATE INDEX IF NOT EXISTS idx_percent_milestones_alert_digest "
1973
+ "ON percent_milestones(week_start_date, percent_threshold, "
1974
+ "reset_event_id, account_key, captured_at_utc, cumulative_cost_usd, "
1975
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
1976
+ ("five_hour_milestones",
1977
+ ("five_hour_window_key", "percent_threshold", "reset_event_id",
1978
+ "account_key", "captured_at_utc", "block_cost_usd", "alerted_at"),
1979
+ "CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_alert_digest "
1980
+ "ON five_hour_milestones(five_hour_window_key, percent_threshold, "
1981
+ "reset_event_id, account_key, captured_at_utc, block_cost_usd, "
1982
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
1983
+ ("budget_milestones",
1984
+ ("vendor", "period_start_at", "period", "threshold", "account_key",
1985
+ "budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
1986
+ "alerted_at"),
1987
+ "CREATE INDEX IF NOT EXISTS idx_budget_milestones_alert_digest "
1988
+ "ON budget_milestones(vendor, period_start_at, period, threshold, "
1989
+ "account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
1990
+ "alerted_at) WHERE vendor <> 'codex' AND alerted_at IS NOT NULL"),
1991
+ ("projected_milestones",
1992
+ ("week_start_at", "period", "metric", "threshold", "account_key",
1993
+ "projected_value", "denominator", "crossed_at_utc", "alerted_at"),
1994
+ "CREATE INDEX IF NOT EXISTS idx_projected_milestones_alert_digest "
1995
+ "ON projected_milestones(week_start_at, period, metric, threshold, "
1996
+ "account_key, projected_value, denominator, crossed_at_utc, alerted_at) "
1997
+ "WHERE metric <> 'codex_budget_usd' AND alerted_at IS NOT NULL"),
1998
+ ("project_budget_milestones",
1999
+ ("week_start_at", "project_key", "threshold", "account_key",
2000
+ "budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
2001
+ "alerted_at"),
2002
+ "CREATE INDEX IF NOT EXISTS idx_project_budget_milestones_alert_digest "
2003
+ "ON project_budget_milestones(week_start_at, project_key, threshold, "
2004
+ "account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
2005
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
2006
+ ("weekly_usage_snapshots",
2007
+ ("week_start_at", "week_end_at", "week_start_date", "account_key",
2008
+ "weekly_observation_held", "captured_at_utc"),
2009
+ "CREATE INDEX IF NOT EXISTS idx_usage_week_boundary_group "
2010
+ "ON weekly_usage_snapshots(week_start_at, week_end_at, week_start_date, "
2011
+ "account_key, weekly_observation_held, captured_at_utc) "
2012
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
2013
+ ("weekly_usage_snapshots",
2014
+ ("week_start_date", "week_end_date", "account_key",
2015
+ "weekly_observation_held", "captured_at_utc"),
2016
+ "CREATE INDEX IF NOT EXISTS idx_usage_week_date_group "
2017
+ "ON weekly_usage_snapshots(week_start_date, week_end_date, account_key, "
2018
+ "weekly_observation_held, captured_at_utc)"),
2019
+ # Amendment 1 (binding Codex answer `c901-design-2`): the recurring history
2020
+ # reads the #901 plan guard escalated. Item 2: `_load_codex_cycles` orders
2021
+ # every weekly block of the active roots by reset; the partial predicate
2022
+ # deliberately keeps orphaned rows so `include_orphaned=True` streams too.
2023
+ # Amendment 1b widened it past the reset second: `_canonicalize_codex_
2024
+ # cluster` keeps the first maximal member, so the order of rows tied on one
2025
+ # second is observable, and the trailing columns make it explicit. The
2026
+ # table's UNIQUE constraint makes them unique within the population.
2027
+ # Amendment 1c (`c901-design-3`): the legacy plan fed its sorter in two
2028
+ # different orders. A read of two or more roots scanned
2029
+ # `idx_quota_blocks_active` (ties by root, account, orphaned_at, limit,
2030
+ # slot, reset spelling); a read of exactly one root seeked the UNIQUE
2031
+ # autoindex (the same order without orphaned_at). The first index below
2032
+ # serves the multi-root read, the second the single-root read, each in its
2033
+ # legacy tie order.
2034
+ ("quota_window_blocks",
2035
+ ("source", "window_minutes", "resets_at_utc", "source_root_key",
2036
+ "account_key", "orphaned_at", "logical_limit_key", "observed_slot"),
2037
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_reset_order "
2038
+ "ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
2039
+ "source_root_key ASC, account_key ASC, orphaned_at ASC, "
2040
+ "logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
2041
+ "WHERE source='codex' AND window_minutes=10080"),
2042
+ ("quota_window_blocks",
2043
+ ("source", "window_minutes", "resets_at_utc", "source_root_key",
2044
+ "account_key", "logical_limit_key", "observed_slot"),
2045
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_single_root_order "
2046
+ "ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
2047
+ "source_root_key ASC, account_key ASC, "
2048
+ "logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
2049
+ "WHERE source='codex' AND window_minutes=10080"),
2050
+ # Item 3: `_navigable_claude_refs` de-duplicates every milestone week; a
2051
+ # full index, because the alert digest index cannot see unalerted rows.
2052
+ ("percent_milestones", ("week_start_date",),
2053
+ "CREATE INDEX IF NOT EXISTS idx_percent_milestones_week_date "
2054
+ "ON percent_milestones(week_start_date ASC)"),
2055
+ # Item 4: `_historic_root_keys` reads the complete relations.
2056
+ ("quota_percent_milestones", ("source_root_key",),
2057
+ "CREATE INDEX IF NOT EXISTS idx_quota_milestones_root "
2058
+ "ON quota_percent_milestones(source_root_key ASC)"),
2059
+ ("quota_threshold_events", ("source_root_key",),
2060
+ "CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_root "
2061
+ "ON quota_threshold_events(source_root_key ASC)"),
2062
+ # Item 5 (and item 4's quota_window_blocks leg): one root's distinct
2063
+ # (group key, group digest) pairs stream after three equality columns.
2064
+ ("quota_window_blocks",
2065
+ ("source_root_key", "source", "orphaned_at", "physical_group_key",
2066
+ "physical_group_digest"),
2067
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_root_group_pairs "
2068
+ "ON quota_window_blocks(source_root_key ASC, source ASC, orphaned_at ASC, "
2069
+ "physical_group_key ASC, physical_group_digest ASC)"),
2070
+ # Items 6 and 7: the WeekRef reset-event applier and
2071
+ # `in_place_cut_instants` read every event in the chokepoint's order.
2072
+ ("week_reset_events", ("effective_reset_at_utc", "id"),
2073
+ "CREATE INDEX IF NOT EXISTS idx_week_reset_events_effective_order "
2074
+ "ON week_reset_events(unixepoch(effective_reset_at_utc) DESC, id DESC)"),
2075
+ # Item 9: `_compute_subscription_weeks` scans representative rows in
2076
+ # (minimum start, date) order and picks and aggregates each date's
2077
+ # qualifying population. The order index's predicate omits the answer's
2078
+ # `week_start_date IS NOT NULL`: the column is NOT NULL, so SQLite drops
2079
+ # that term from the outer scan's WHERE and then cannot prove the partial
2080
+ # index usable ("no query solution" under INDEXED BY, measured). The
2081
+ # indexed population is identical. The pick index keeps the term, because
2082
+ # every lookup through it carries a date equality that implies it.
2083
+ ("weekly_usage_snapshots",
2084
+ ("week_start_at", "week_end_at", "week_start_date"),
2085
+ "CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_order "
2086
+ "ON weekly_usage_snapshots(week_start_at ASC, week_start_date ASC) "
2087
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
2088
+ ("weekly_usage_snapshots",
2089
+ ("week_start_date", "week_start_at", "week_end_at", "id"),
2090
+ "CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_pick "
2091
+ "ON weekly_usage_snapshots(week_start_date ASC, week_start_at ASC, id ASC) "
2092
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL "
2093
+ "AND week_start_date IS NOT NULL"),
2094
+ )
2095
+
2096
+
2097
+ def _apply_stats_read_indexes(conn: sqlite3.Connection) -> None:
2098
+ """Create each #901 read index whose table and columns exist (epoch 1017).
2099
+
2100
+ ``open_db`` calls this twice: beside the quota projection schema, and
2101
+ again after the stats migration dispatcher, because a legacy store being
2102
+ cut over gains one indexed column (``reset_event_id``, migration 005) only
2103
+ from the dispatcher. Every statement is IF NOT EXISTS, so the second call
2104
+ creates exactly what the first had to skip, and an index is never attempted
2105
+ over a column that is not there yet.
2106
+ """
2107
+ for table, required, statement in _STATS_READ_INDEX_DDL:
2108
+ columns = {
2109
+ str(row[0]) for row in conn.execute(
2110
+ "SELECT name FROM pragma_table_info(?)", (table,))
2111
+ }
2112
+ if set(required) <= columns:
2113
+ conn.execute(statement)
2114
+
2115
+
1864
2116
  def open_db(*, _target_path=None) -> sqlite3.Connection:
1865
2117
  # ``_target_path`` (internal, keyword-only) builds/opens the stats index at
1866
2118
  # an ALTERNATE path instead of the module-global ``DB_PATH`` — the seam
@@ -2129,6 +2381,10 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2129
2381
  conn, _STATS_MIGRATIONS, "stats.db"
2130
2382
  )
2131
2383
  return conn
2384
+ # PR-3: everything below is pending schema, migration or cutover work;
2385
+ # it sorts under FILE, and the writer temp store is selected again
2386
+ # before the connection is returned.
2387
+ _cctally_store.apply_schema_work_temp_store(conn)
2132
2388
  _fixups_current = _cctally_store.stats_open_fixups_current(conn)
2133
2389
  conn.execute(
2134
2390
  """
@@ -2854,6 +3110,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2854
3110
  # without running its handler, so this open-time call is the sole creator).
2855
3111
  if not _fixups_current:
2856
3112
  _apply_quota_projection_schema(conn)
3113
+ _apply_stats_read_indexes(conn)
2857
3114
 
2858
3115
  # Migration framework dispatcher. Replaces the prior inline gate stack
2859
3116
  # (has_blocks + _migration_done) with the framework's _run_pending_-
@@ -2871,6 +3128,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2871
3128
  _run_pending_migrations(
2872
3129
  conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
2873
3130
  )
3131
+ _apply_stats_read_indexes(conn)
2874
3132
 
2875
3133
  # One-time historical backfill of five_hour_blocks (rollup only;
2876
3134
  # milestones are forward-only per spec §4.3 / [Write-once milestones]).
@@ -3089,6 +3347,106 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
3089
3347
  "cutover_seen INTEGER NOT NULL DEFAULT 0, "
3090
3348
  "cutover_account_key TEXT)"
3091
3349
  )
3350
+ # #825/#832 epoch 1018 (spec section 8): the bounded, disposable
3351
+ # summary of the complete journal records the admission predicate
3352
+ # EXCLUDED (post-cutover records without the current writer
3353
+ # generation). Ingest extends it inside its cycle transaction and a
3354
+ # rebuild reproduces it from the pinned prefix; `complete_through` is
3355
+ # NULL when the row was started mid-journal (it then describes only
3356
+ # what it saw). Coordinates are JSON `{"segment", "offset"}` text, and
3357
+ # `coordinate_digest` is the order-independent population digest of
3358
+ # every excluded coordinate (`_lib_account_protocol`).
3359
+ conn.execute(
3360
+ "CREATE TABLE IF NOT EXISTS journal_admission_summary ("
3361
+ "id INTEGER PRIMARY KEY CHECK (id = 1), "
3362
+ "cutover_id TEXT, "
3363
+ "scanned_high_water TEXT, "
3364
+ "excluded_total INTEGER NOT NULL DEFAULT 0, "
3365
+ "family_counts_json TEXT NOT NULL DEFAULT '{}', "
3366
+ "first_excluded_coordinate TEXT, "
3367
+ "latest_excluded_coordinate TEXT, "
3368
+ "coordinate_digest TEXT, "
3369
+ "complete_through TEXT, "
3370
+ # The boundary payload this summary classified against, so the
3371
+ # live ingest path resolves admission without scanning the journal.
3372
+ "boundary_json TEXT)"
3373
+ )
3374
+ # #825/#832 fix round A (Codex fix-round-1 Q1): the derivation
3375
+ # frontier. Every admitted derivable input at or before this journal
3376
+ # coordinate has had its decisions journaled by a current binary. Live
3377
+ # ingest advances it with the cursor only while the two are equal; a
3378
+ # rebuild carries it from a trusted predecessor index or sets it after
3379
+ # the shared completion step proved the window. Absent means unknown,
3380
+ # which the completion step reads as "replay from the cutover".
3381
+ # Fix round C (P1-1): the frontier counts CONSUMED INPUTS, and the
3382
+ # decisions a cycle derives from them land beyond it, so the row also
3383
+ # names the output end — the coordinate through which every decision
3384
+ # derived from an input at or before the frontier is journaled, with
3385
+ # nothing between the two derived from a later input. NULL means
3386
+ # nothing is owed beyond the frontier.
3387
+ conn.execute(
3388
+ "CREATE TABLE IF NOT EXISTS journal_derivation_frontier ("
3389
+ "id INTEGER PRIMARY KEY CHECK (id = 1), "
3390
+ "segment TEXT NOT NULL, "
3391
+ "offset INTEGER NOT NULL, "
3392
+ "output_segment TEXT, "
3393
+ "output_offset INTEGER)"
3394
+ )
3395
+ # An index an earlier build of this epoch created lacks the two
3396
+ # columns; additive, so it keeps its frontier row.
3397
+ add_column_if_missing(
3398
+ conn, "journal_derivation_frontier", "output_segment", "TEXT")
3399
+ add_column_if_missing(
3400
+ conn, "journal_derivation_frontier", "output_offset", "INTEGER")
3401
+ # Revision 2 (#946, Codex design-fix-round-3 Q5): the proof
3402
+ # generation. A row written under the old traversal rule (NULL here)
3403
+ # may name a valid endpoint past an input nothing derived, so only
3404
+ # `_cctally_journal.DERIVATION_PROOF_GENERATION` is trusted.
3405
+ add_column_if_missing(
3406
+ conn, "journal_derivation_frontier", "proof_generation", "INTEGER")
3407
+ # #825/#832 fix round A (Codex fix-round-1 Q2): the physical revision
3408
+ # inventory of every EXCLUDED event or correction action the index has
3409
+ # consumed, by logical id. Admission hides these from effective
3410
+ # selection, so without it a current writer would classify the id NEW
3411
+ # and append a revision-0 twin the previous release reads as a
3412
+ # same-revision conflict.
3413
+ conn.execute(
3414
+ "CREATE TABLE IF NOT EXISTS journal_excluded_event_revisions ("
3415
+ "event_id TEXT PRIMARY KEY, "
3416
+ "max_rev INTEGER NOT NULL)"
3417
+ )
3418
+ # #825/#832 epoch 1018 (spec 5.6 item 9): the outcome quarantine. A
3419
+ # deterministic function of journal order that ingest extends inside
3420
+ # its cycle transaction and a rebuild reproduces from the pinned
3421
+ # prefix: one row per uncertainty trigger (an excluded record or a
3422
+ # current `legacy_spool_activity` op, at its physical coordinate), the
3423
+ # whole-prefix envelope per scope ("excluded" applies to every
3424
+ # account), and each account's accounting and control certificates
3425
+ # with the trigger sequence they were granted against. A certificate
3426
+ # granted against an older sequence is void; a missing or
3427
+ # unverifiable row reads as quarantined.
3428
+ conn.execute(
3429
+ "CREATE TABLE IF NOT EXISTS outcome_quarantine_triggers ("
3430
+ "seq INTEGER PRIMARY KEY, "
3431
+ "segment TEXT, "
3432
+ "byte_offset INTEGER, "
3433
+ "kind TEXT NOT NULL, "
3434
+ "reason TEXT NOT NULL, "
3435
+ "at TEXT)"
3436
+ )
3437
+ conn.execute(
3438
+ "CREATE TABLE IF NOT EXISTS outcome_quarantine_envelope ("
3439
+ "scope TEXT PRIMARY KEY, "
3440
+ "envelope_json TEXT NOT NULL)"
3441
+ )
3442
+ conn.execute(
3443
+ "CREATE TABLE IF NOT EXISTS outcome_quarantine_certificates ("
3444
+ "account_key TEXT PRIMARY KEY, "
3445
+ "accounting_certificate_json TEXT, "
3446
+ "accounting_trigger_seq INTEGER, "
3447
+ "control_certificate_json TEXT, "
3448
+ "control_trigger_seq INTEGER)"
3449
+ )
3092
3450
  conn.execute(
3093
3451
  "CREATE TABLE IF NOT EXISTS journal_selector_batches ("
3094
3452
  "batch_id TEXT PRIMARY KEY, "
@@ -3148,6 +3506,45 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
3148
3506
  "started_at_utc TEXT NOT NULL, "
3149
3507
  "stamped_at_utc TEXT NOT NULL)"
3150
3508
  )
3509
+ # #825/#832 Task 4.0 (epoch 1018): a milestone whose snapshot a
3510
+ # credit's stale-replica removal deleted. The removal's band starts at
3511
+ # floor10 of the arming tick, so it also reaches the GENUINE pre-credit
3512
+ # peak tick when that tick shares the arming tick's ten-minute slot —
3513
+ # and that tick is the one that crossed the last weekly and five-hour
3514
+ # milestones. A rebuild folds the suppression (order 30) before the
3515
+ # milestones (order 60), so their journaled `usage_snapshot_ref` named
3516
+ # a row that no longer existed and the NOT NULL row was silently
3517
+ # dropped; the next tick then re-recorded the highest weekly milestone
3518
+ # and its conflict convergence raised on every cycle.
3519
+ #
3520
+ # The tombstone records every journaled snapshot the index deleted —
3521
+ # the live stale-replica removals and the suppression appliers alike,
3522
+ # because a trigger sees every DELETE — so the fold's ONE resolver
3523
+ # (`_cctally_journal._resolve_fold_ref`) can tell "a suppression
3524
+ # removed it" from a genuinely unresolvable reference. The trigger's
3525
+ # second half points every milestone that referenced the deleted row at
3526
+ # the 0 "no snapshot row" sentinel, which is what the rebuild resolves
3527
+ # such a reference to, so the live index and a rebuild agree. Both
3528
+ # milestone readers already treat 0 as "no row": the five-hour
3529
+ # milestone read LEFT JOINs (and so publishes the effective weekly
3530
+ # value as unavailable, as it already did for the dangling rowid), and
3531
+ # the observation-gap classifier never continues a run through 0.
3532
+ conn.execute(
3533
+ "CREATE TABLE IF NOT EXISTS usage_snapshot_suppressions ("
3534
+ "journal_id TEXT PRIMARY KEY)"
3535
+ )
3536
+ conn.execute(
3537
+ "CREATE TRIGGER IF NOT EXISTS trg_usage_snapshot_suppression "
3538
+ "AFTER DELETE ON weekly_usage_snapshots "
3539
+ "BEGIN "
3540
+ "INSERT OR IGNORE INTO usage_snapshot_suppressions (journal_id) "
3541
+ "SELECT OLD.journal_id WHERE OLD.journal_id IS NOT NULL; "
3542
+ "UPDATE percent_milestones SET usage_snapshot_id = 0 "
3543
+ "WHERE usage_snapshot_id = OLD.id; "
3544
+ "UPDATE five_hour_milestones SET usage_snapshot_id = 0 "
3545
+ "WHERE usage_snapshot_id = OLD.id; "
3546
+ "END"
3547
+ )
3151
3548
 
3152
3549
  # §6.2 backfill gate (Task 8): stamp the one-shot marker AFTER the three
3153
3550
  # open-time backfills ran, so the next open skips them (and their probes)
@@ -3186,6 +3583,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
3186
3583
  except Exception:
3187
3584
  pass
3188
3585
  raise
3586
+ _cctally_store.apply_writer_temp_store(conn)
3189
3587
  return conn
3190
3588
 
3191
3589