cctally 1.113.0 → 1.115.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +47 -0
- package/README.md +4 -6
- package/bin/_cctally_account.py +32 -8
- package/bin/_cctally_cache.py +6035 -722
- package/bin/_cctally_completion.py +1287 -0
- package/bin/_cctally_config.py +37 -0
- package/bin/_cctally_core.py +400 -2
- package/bin/_cctally_dashboard.py +98 -20
- package/bin/_cctally_dashboard_conversation.py +292 -23
- package/bin/_cctally_dashboard_envelope.py +16 -3
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +285 -213
- package/bin/_cctally_db.py +1177 -23
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +572 -45
- package/bin/_cctally_five_hour.py +84 -33
- package/bin/_cctally_forecast.py +108 -15
- package/bin/_cctally_journal.py +3950 -116
- package/bin/_cctally_journal_repair.py +101 -12
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_project.py +8 -2
- package/bin/_cctally_quarantine.py +652 -0
- package/bin/_cctally_quota.py +227 -37
- package/bin/_cctally_quota_model.py +360 -83
- package/bin/_cctally_record.py +691 -149
- package/bin/_cctally_rederive.py +320 -41
- package/bin/_cctally_refresh.py +84 -13
- package/bin/_cctally_source_analytics.py +83 -17
- package/bin/_cctally_statusline.py +6154 -517
- package/bin/_cctally_store.py +367 -53
- package/bin/_cctally_transcript.py +8 -7
- package/bin/_cctally_tui.py +61 -56
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_account_binding.py +152 -0
- package/bin/_lib_account_protocol.py +386 -0
- package/bin/_lib_aggregators.py +4 -0
- package/bin/_lib_cache_coverage.py +13 -1
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_cache_writer_lock.py +19 -3
- package/bin/_lib_clone_copy.py +142 -0
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +297 -49
- package/bin/_lib_codex_conversation_watch.py +59 -14
- package/bin/_lib_codex_drain_admission.py +214 -0
- package/bin/_lib_codex_metadata.py +18 -1
- package/bin/_lib_codex_publication_barrier.py +277 -0
- package/bin/_lib_codex_rollout_container.py +502 -0
- package/bin/_lib_conversation_dispatch.py +3 -0
- package/bin/_lib_conversation_query.py +106 -59
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_detector_access_registry.py +3817 -0
- package/bin/_lib_doctor.py +714 -69
- package/bin/_lib_five_hour.py +126 -0
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_ingest_frontier.py +414 -20
- package/bin/_lib_journal.py +141 -10
- package/bin/_lib_jsonl.py +5 -4
- package/bin/_lib_legacy_spool.py +126 -0
- package/bin/_lib_legacy_writer_evidence.py +136 -0
- package/bin/_lib_merge_gate.py +231 -86
- package/bin/_lib_meter_rate_change.py +10 -0
- package/bin/_lib_outcome_quarantine.py +690 -0
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +199 -9
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_quarantine.py +154 -0
- package/bin/_lib_quota_calibration.py +53 -8
- package/bin/_lib_quota_copy.py +2 -1
- package/bin/_lib_quota_model.py +288 -46
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +319 -82
- package/bin/_lib_segment_summary.py +35 -2
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_statusline_candidates.py +292 -23
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +31 -1
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +19 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_cctally_config.py
CHANGED
|
@@ -529,6 +529,43 @@ def resolve_retention_days(config: dict) -> int:
|
|
|
529
529
|
return default
|
|
530
530
|
|
|
531
531
|
|
|
532
|
+
#: The test-only switch (#965). Honored only at exactly "1", only when no
|
|
533
|
+
#: retention setting exists, and only for code running from a git checkout.
|
|
534
|
+
CONVERSATION_RETENTION_TEST_SWITCH = "CCTALLY_TEST_CONVERSATION_RETENTION_DEFAULT_OFF"
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _conversation_retention_setting_absent(config) -> bool:
|
|
538
|
+
"""True when no ``conversation.retention_days`` setting exists (#965 5.2).
|
|
539
|
+
|
|
540
|
+
Absent: no ``conversation`` key, or a ``conversation`` dictionary without
|
|
541
|
+
``retention_days``. A present key of any value (``null`` and malformed
|
|
542
|
+
values included), a non-dictionary ``conversation`` and a non-dictionary
|
|
543
|
+
config are not absent: they keep ``resolve_retention_days``'s meaning."""
|
|
544
|
+
if not isinstance(config, dict):
|
|
545
|
+
return False
|
|
546
|
+
if "conversation" not in config:
|
|
547
|
+
return True
|
|
548
|
+
block = config["conversation"]
|
|
549
|
+
return isinstance(block, dict) and "retention_days" not in block
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def resolve_prune_retention_days(config: dict) -> int:
|
|
553
|
+
"""Operational transcript-retention window of the three prune paths (#965).
|
|
554
|
+
|
|
555
|
+
``resolve_retention_days`` stays the reporting resolver (``config get``,
|
|
556
|
+
the settings manifest, docs). This one differs in exactly one case: with
|
|
557
|
+
no retention setting, the test switch at exactly ``"1"`` and a ``.git``
|
|
558
|
+
entry at the running code's root, it returns ``0`` (keep forever). The
|
|
559
|
+
switch and the marker are read on every call; nothing is cached, written
|
|
560
|
+
or logged. docs/fixtures-gotchas.md, "Conversation retention is off in
|
|
561
|
+
test environments (#965)"."""
|
|
562
|
+
if (_conversation_retention_setting_absent(config)
|
|
563
|
+
and os.environ.get(CONVERSATION_RETENTION_TEST_SWITCH) == "1"
|
|
564
|
+
and _cctally_core._has_git_checkout_marker()):
|
|
565
|
+
return 0
|
|
566
|
+
return resolve_retention_days(config)
|
|
567
|
+
|
|
568
|
+
|
|
532
569
|
def _config_codex_leaf_value(config: dict, key: str) -> object:
|
|
533
570
|
"""Read one nested Codex budget leaf with its public default.
|
|
534
571
|
|
package/bin/_cctally_core.py
CHANGED
|
@@ -76,6 +76,8 @@ def _init_paths_from_env() -> None:
|
|
|
76
76
|
global STATUSLINE_CANDIDATE_DIR, STATUSLINE_SELECTED_PATH
|
|
77
77
|
global STATUSLINE_TRANSPORT_MARKER_PATH
|
|
78
78
|
global STATUSLINE_AUTHORITATIVE_7D_PATH, STATUSLINE_AUTHORITATIVE_5H_PATH
|
|
79
|
+
global STATUSLINE_ACCOUNT_BOUND_DIR, LEGACY_SPOOL_BASELINE_PATH
|
|
80
|
+
global ACCOUNT_BOUND_CUTOVER_CACHE_PATH, LEGACY_WRITER_EVIDENCE_PATH
|
|
79
81
|
global OAUTH_BACKOFF_MARKER_PATH, OAUTH_BACKOFF_COUNT_PATH
|
|
80
82
|
global UPDATE_STATE_PATH, UPDATE_SUPPRESS_PATH
|
|
81
83
|
global UPDATE_LOCK_PATH, UPDATE_LOG_PATH, UPDATE_LOG_ROTATED_PATH
|
|
@@ -184,6 +186,19 @@ def _init_paths_from_env() -> None:
|
|
|
184
186
|
STATUSLINE_TRANSPORT_MARKER_PATH = APP_DIR / "statusline-transport.last"
|
|
185
187
|
STATUSLINE_AUTHORITATIVE_7D_PATH = APP_DIR / "statusline-authoritative-7d.json"
|
|
186
188
|
STATUSLINE_AUTHORITATIVE_5H_PATH = APP_DIR / "statusline-authoritative-5h.json"
|
|
189
|
+
# #825/#832 account-bound protocol (spec section 7). Every current
|
|
190
|
+
# coordination file lives under its own sibling root; the legacy paths
|
|
191
|
+
# above are never read as authority after the cutover, only compared with
|
|
192
|
+
# the journal-anchored legacy spool baseline. The coordinate cache only
|
|
193
|
+
# accelerates locating the cutover op and never confers authority; the
|
|
194
|
+
# legacy-writer evidence sidecar is diagnostic only (spec section 8).
|
|
195
|
+
STATUSLINE_ACCOUNT_BOUND_DIR = APP_DIR / "statusline-account-bound-v1"
|
|
196
|
+
LEGACY_SPOOL_BASELINE_PATH = (
|
|
197
|
+
STATUSLINE_ACCOUNT_BOUND_DIR / "legacy-spool-baseline.json")
|
|
198
|
+
ACCOUNT_BOUND_CUTOVER_CACHE_PATH = (
|
|
199
|
+
STATUSLINE_ACCOUNT_BOUND_DIR / "cutover-coordinate.json")
|
|
200
|
+
LEGACY_WRITER_EVIDENCE_PATH = (
|
|
201
|
+
STATUSLINE_ACCOUNT_BOUND_DIR / "legacy-writer-evidence.json")
|
|
187
202
|
OAUTH_BACKOFF_MARKER_PATH = APP_DIR / "oauth-backoff.until"
|
|
188
203
|
# Consecutive-429 counter (text int) driving the headerless exponential
|
|
189
204
|
# backoff (base * 2**count). Separate from the deadline marker so the
|
|
@@ -261,6 +276,18 @@ def _repo_root() -> pathlib.Path:
|
|
|
261
276
|
return pathlib.Path(__file__).resolve().parent.parent
|
|
262
277
|
|
|
263
278
|
|
|
279
|
+
def _has_git_checkout_marker() -> bool:
|
|
280
|
+
"""True iff the running code has a ``.git`` entry at its root (#965 5.1).
|
|
281
|
+
|
|
282
|
+
A main checkout has a ``.git`` directory and a linked worktree a ``.git``
|
|
283
|
+
file; both count. Eligibility follows the code's resolved location
|
|
284
|
+
(``_repo_root()``), never the current directory, and is evaluated on every
|
|
285
|
+
call. No suppressor (``CCTALLY_DISABLE_DEV_AUTODETECT``,
|
|
286
|
+
``CCTALLY_ALLOW_PROD_MIGRATION``) affects it. npm and Homebrew installs
|
|
287
|
+
ship without ``.git``, so they never read True."""
|
|
288
|
+
return (_repo_root() / ".git").exists()
|
|
289
|
+
|
|
290
|
+
|
|
264
291
|
def _is_dev_checkout() -> bool:
|
|
265
292
|
"""True iff running from a git checkout (a ``.git`` entry at the repo
|
|
266
293
|
root — a directory for a main checkout, a file for a worktree) AND the
|
|
@@ -272,7 +299,7 @@ def _is_dev_checkout() -> bool:
|
|
|
272
299
|
install copies ship without ``.git`` so they never read True."""
|
|
273
300
|
if _truthy_env("CCTALLY_DISABLE_DEV_AUTODETECT"):
|
|
274
301
|
return False
|
|
275
|
-
return (
|
|
302
|
+
return _has_git_checkout_marker()
|
|
276
303
|
|
|
277
304
|
|
|
278
305
|
def is_preview_channel() -> bool:
|
|
@@ -473,9 +500,33 @@ _init_paths_from_env()
|
|
|
473
500
|
# tick must carry that earlier evidence into the journal, even after this
|
|
474
501
|
# binary learns a new model card. A pre-existing open projection that was
|
|
475
502
|
# rebuilt without the marker remains unproven until a new tick recomputes it.
|
|
476
|
-
|
|
503
|
+
# 1016 -> 1017 (#901 §5.3, T3): read indexes only. Every stats digest relation
|
|
504
|
+
# (`_lib_dashboard_sources._CODEX_STATS_DIGEST_RELATIONS` and
|
|
505
|
+
# `_CLAUDE_STATS_DIGEST_RELATIONS`) sorted its whole relation through a temp
|
|
506
|
+
# b-tree on every dashboard build, and `_fetch_current_week_snapshots` grouped
|
|
507
|
+
# every boundary-aware and legacy-date snapshot the same way; each now streams
|
|
508
|
+
# through an index in its own order (`_STATS_READ_INDEX_DDL`). No table or
|
|
509
|
+
# column changes and no digest byte moves. The same epoch carries Amendment 1's
|
|
510
|
+
# history-read indexes (items 2-7 and 9, `c901-design-2`, with item 2's second
|
|
511
|
+
# single-root index from Amendment 1c, `c901-design-3`): every full dashboard
|
|
512
|
+
# build enumerated weekly blocks, milestone weeks, root keys, group pairs,
|
|
513
|
+
# reset events and subscription anchors through temp b-trees. The mechanical
|
|
514
|
+
# reason this is a bump and not a migration is unchanged: the registry is
|
|
515
|
+
# frozen at 13, and an epoch-current open returns before any schema work, so a
|
|
516
|
+
# schema helper alone would never run on an upgraded install.
|
|
517
|
+
# 1017 -> 1018 (#825/#832 spec section 6): the same journal now has a different
|
|
518
|
+
# authoritative interpretation — the account-bound admission predicate excludes
|
|
519
|
+
# unstamped post-cutover records before any derivation — so an index
|
|
520
|
+
# materialized under the old interpretation is not current even though its
|
|
521
|
+
# schema is usable. The epoch also carries the disposable
|
|
522
|
+
# `journal_admission_summary` table, whose presence is the protocol marker
|
|
523
|
+
# `_cctally_store.validated_read_snapshot` checks inside each decision
|
|
524
|
+
# scope's own snapshot. An epoch transition, not a numbered migration: the
|
|
525
|
+
# registry stays frozen at 13.
|
|
526
|
+
STATS_INDEX_EPOCH = 1018
|
|
477
527
|
LEGACY_STATS_HEAD = 13
|
|
478
528
|
|
|
529
|
+
|
|
479
530
|
#: #496 S1 F1. A NEW branch, for a state that cannot occur before the
|
|
480
531
|
#: publication transaction exists: a replacement index was published and then
|
|
481
532
|
#: failed validation on a fresh connection. The existing corrupt-stats text
|
|
@@ -1861,6 +1912,207 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
|
|
|
1861
1912
|
"next_evaluation_by_root_json", "TEXT NOT NULL DEFAULT '{}'")
|
|
1862
1913
|
|
|
1863
1914
|
|
|
1915
|
+
#: #901 §5.3 (T3), epoch 1017: (table, columns the index reads, DDL). Each
|
|
1916
|
+
#: index's leading columns are the statement's equality filter (when it has
|
|
1917
|
+
#: one) followed by its ORDER BY / GROUP BY list, so the statement streams off
|
|
1918
|
+
#: the index with no sorter; the digest indexes are covering, because every
|
|
1919
|
+
#: digest relation selects exactly its ordering columns. SQLite stores each
|
|
1920
|
+
#: statement verbatim in `sqlite_schema`, which `_REBUILD_SCHEMA_FINGERPRINT`
|
|
1921
|
+
#: hashes: change the text and the fingerprint must move with it.
|
|
1922
|
+
_STATS_READ_INDEX_DDL: "tuple[tuple[str, tuple[str, ...], str], ...]" = (
|
|
1923
|
+
("quota_projection_state", ("source_root_key", "physical_signature"),
|
|
1924
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_projection_state_digest "
|
|
1925
|
+
"ON quota_projection_state(source_root_key, physical_signature)"),
|
|
1926
|
+
("quota_window_blocks",
|
|
1927
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1928
|
+
"window_minutes", "limit_id", "limit_name", "resets_at_utc",
|
|
1929
|
+
"nominal_start_at_utc", "first_observed_at_utc", "last_observed_at_utc",
|
|
1930
|
+
"first_percent", "current_percent", "orphaned_at"),
|
|
1931
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_digest "
|
|
1932
|
+
"ON quota_window_blocks(source, source_root_key, logical_limit_key, "
|
|
1933
|
+
"observed_slot, window_minutes, limit_id, limit_name, resets_at_utc, "
|
|
1934
|
+
"nominal_start_at_utc, first_observed_at_utc, last_observed_at_utc, "
|
|
1935
|
+
"first_percent, current_percent, orphaned_at) WHERE source='codex'"),
|
|
1936
|
+
("quota_percent_milestones",
|
|
1937
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1938
|
+
"window_minutes", "resets_at_utc", "percent_threshold",
|
|
1939
|
+
"captured_at_utc", "high_water_percent", "orphaned_at"),
|
|
1940
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_milestones_digest "
|
|
1941
|
+
"ON quota_percent_milestones(source, source_root_key, logical_limit_key, "
|
|
1942
|
+
"observed_slot, window_minutes, resets_at_utc, percent_threshold, "
|
|
1943
|
+
"captured_at_utc, high_water_percent, orphaned_at) WHERE source='codex'"),
|
|
1944
|
+
("quota_threshold_events",
|
|
1945
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1946
|
+
"window_minutes", "resets_at_utc", "threshold", "qualifying_kind",
|
|
1947
|
+
"qualifying_percent", "projected_percent", "severity", "created_at_utc",
|
|
1948
|
+
"disposition", "alerted_at", "suppressed_at", "orphaned_at"),
|
|
1949
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_digest "
|
|
1950
|
+
"ON quota_threshold_events(source, source_root_key, logical_limit_key, "
|
|
1951
|
+
"observed_slot, window_minutes, resets_at_utc, threshold, "
|
|
1952
|
+
"qualifying_kind, qualifying_percent, projected_percent, severity, "
|
|
1953
|
+
"created_at_utc, disposition, alerted_at, suppressed_at, orphaned_at) "
|
|
1954
|
+
"WHERE source='codex'"),
|
|
1955
|
+
("budget_milestones",
|
|
1956
|
+
("vendor", "period_start_at", "period", "threshold", "budget_usd",
|
|
1957
|
+
"spent_usd", "consumption_pct", "crossed_at_utc", "alerted_at"),
|
|
1958
|
+
"CREATE INDEX IF NOT EXISTS idx_budget_milestones_codex_digest "
|
|
1959
|
+
"ON budget_milestones(vendor, period_start_at, period, threshold, "
|
|
1960
|
+
"budget_usd, spent_usd, consumption_pct, crossed_at_utc, alerted_at) "
|
|
1961
|
+
"WHERE vendor='codex'"),
|
|
1962
|
+
("projected_milestones",
|
|
1963
|
+
("metric", "week_start_at", "period", "threshold", "projected_value",
|
|
1964
|
+
"denominator", "crossed_at_utc", "alerted_at"),
|
|
1965
|
+
"CREATE INDEX IF NOT EXISTS idx_projected_milestones_codex_digest "
|
|
1966
|
+
"ON projected_milestones(metric, week_start_at, period, threshold, "
|
|
1967
|
+
"projected_value, denominator, crossed_at_utc, alerted_at) "
|
|
1968
|
+
"WHERE metric='codex_budget_usd'"),
|
|
1969
|
+
("percent_milestones",
|
|
1970
|
+
("week_start_date", "percent_threshold", "reset_event_id", "account_key",
|
|
1971
|
+
"captured_at_utc", "cumulative_cost_usd", "alerted_at"),
|
|
1972
|
+
"CREATE INDEX IF NOT EXISTS idx_percent_milestones_alert_digest "
|
|
1973
|
+
"ON percent_milestones(week_start_date, percent_threshold, "
|
|
1974
|
+
"reset_event_id, account_key, captured_at_utc, cumulative_cost_usd, "
|
|
1975
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
1976
|
+
("five_hour_milestones",
|
|
1977
|
+
("five_hour_window_key", "percent_threshold", "reset_event_id",
|
|
1978
|
+
"account_key", "captured_at_utc", "block_cost_usd", "alerted_at"),
|
|
1979
|
+
"CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_alert_digest "
|
|
1980
|
+
"ON five_hour_milestones(five_hour_window_key, percent_threshold, "
|
|
1981
|
+
"reset_event_id, account_key, captured_at_utc, block_cost_usd, "
|
|
1982
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
1983
|
+
("budget_milestones",
|
|
1984
|
+
("vendor", "period_start_at", "period", "threshold", "account_key",
|
|
1985
|
+
"budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
|
|
1986
|
+
"alerted_at"),
|
|
1987
|
+
"CREATE INDEX IF NOT EXISTS idx_budget_milestones_alert_digest "
|
|
1988
|
+
"ON budget_milestones(vendor, period_start_at, period, threshold, "
|
|
1989
|
+
"account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
|
|
1990
|
+
"alerted_at) WHERE vendor <> 'codex' AND alerted_at IS NOT NULL"),
|
|
1991
|
+
("projected_milestones",
|
|
1992
|
+
("week_start_at", "period", "metric", "threshold", "account_key",
|
|
1993
|
+
"projected_value", "denominator", "crossed_at_utc", "alerted_at"),
|
|
1994
|
+
"CREATE INDEX IF NOT EXISTS idx_projected_milestones_alert_digest "
|
|
1995
|
+
"ON projected_milestones(week_start_at, period, metric, threshold, "
|
|
1996
|
+
"account_key, projected_value, denominator, crossed_at_utc, alerted_at) "
|
|
1997
|
+
"WHERE metric <> 'codex_budget_usd' AND alerted_at IS NOT NULL"),
|
|
1998
|
+
("project_budget_milestones",
|
|
1999
|
+
("week_start_at", "project_key", "threshold", "account_key",
|
|
2000
|
+
"budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
|
|
2001
|
+
"alerted_at"),
|
|
2002
|
+
"CREATE INDEX IF NOT EXISTS idx_project_budget_milestones_alert_digest "
|
|
2003
|
+
"ON project_budget_milestones(week_start_at, project_key, threshold, "
|
|
2004
|
+
"account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
|
|
2005
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
2006
|
+
("weekly_usage_snapshots",
|
|
2007
|
+
("week_start_at", "week_end_at", "week_start_date", "account_key",
|
|
2008
|
+
"weekly_observation_held", "captured_at_utc"),
|
|
2009
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_week_boundary_group "
|
|
2010
|
+
"ON weekly_usage_snapshots(week_start_at, week_end_at, week_start_date, "
|
|
2011
|
+
"account_key, weekly_observation_held, captured_at_utc) "
|
|
2012
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
|
|
2013
|
+
("weekly_usage_snapshots",
|
|
2014
|
+
("week_start_date", "week_end_date", "account_key",
|
|
2015
|
+
"weekly_observation_held", "captured_at_utc"),
|
|
2016
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_week_date_group "
|
|
2017
|
+
"ON weekly_usage_snapshots(week_start_date, week_end_date, account_key, "
|
|
2018
|
+
"weekly_observation_held, captured_at_utc)"),
|
|
2019
|
+
# Amendment 1 (binding Codex answer `c901-design-2`): the recurring history
|
|
2020
|
+
# reads the #901 plan guard escalated. Item 2: `_load_codex_cycles` orders
|
|
2021
|
+
# every weekly block of the active roots by reset; the partial predicate
|
|
2022
|
+
# deliberately keeps orphaned rows so `include_orphaned=True` streams too.
|
|
2023
|
+
# Amendment 1b widened it past the reset second: `_canonicalize_codex_
|
|
2024
|
+
# cluster` keeps the first maximal member, so the order of rows tied on one
|
|
2025
|
+
# second is observable, and the trailing columns make it explicit. The
|
|
2026
|
+
# table's UNIQUE constraint makes them unique within the population.
|
|
2027
|
+
# Amendment 1c (`c901-design-3`): the legacy plan fed its sorter in two
|
|
2028
|
+
# different orders. A read of two or more roots scanned
|
|
2029
|
+
# `idx_quota_blocks_active` (ties by root, account, orphaned_at, limit,
|
|
2030
|
+
# slot, reset spelling); a read of exactly one root seeked the UNIQUE
|
|
2031
|
+
# autoindex (the same order without orphaned_at). The first index below
|
|
2032
|
+
# serves the multi-root read, the second the single-root read, each in its
|
|
2033
|
+
# legacy tie order.
|
|
2034
|
+
("quota_window_blocks",
|
|
2035
|
+
("source", "window_minutes", "resets_at_utc", "source_root_key",
|
|
2036
|
+
"account_key", "orphaned_at", "logical_limit_key", "observed_slot"),
|
|
2037
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_reset_order "
|
|
2038
|
+
"ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
|
|
2039
|
+
"source_root_key ASC, account_key ASC, orphaned_at ASC, "
|
|
2040
|
+
"logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
|
|
2041
|
+
"WHERE source='codex' AND window_minutes=10080"),
|
|
2042
|
+
("quota_window_blocks",
|
|
2043
|
+
("source", "window_minutes", "resets_at_utc", "source_root_key",
|
|
2044
|
+
"account_key", "logical_limit_key", "observed_slot"),
|
|
2045
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_single_root_order "
|
|
2046
|
+
"ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
|
|
2047
|
+
"source_root_key ASC, account_key ASC, "
|
|
2048
|
+
"logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
|
|
2049
|
+
"WHERE source='codex' AND window_minutes=10080"),
|
|
2050
|
+
# Item 3: `_navigable_claude_refs` de-duplicates every milestone week; a
|
|
2051
|
+
# full index, because the alert digest index cannot see unalerted rows.
|
|
2052
|
+
("percent_milestones", ("week_start_date",),
|
|
2053
|
+
"CREATE INDEX IF NOT EXISTS idx_percent_milestones_week_date "
|
|
2054
|
+
"ON percent_milestones(week_start_date ASC)"),
|
|
2055
|
+
# Item 4: `_historic_root_keys` reads the complete relations.
|
|
2056
|
+
("quota_percent_milestones", ("source_root_key",),
|
|
2057
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_milestones_root "
|
|
2058
|
+
"ON quota_percent_milestones(source_root_key ASC)"),
|
|
2059
|
+
("quota_threshold_events", ("source_root_key",),
|
|
2060
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_root "
|
|
2061
|
+
"ON quota_threshold_events(source_root_key ASC)"),
|
|
2062
|
+
# Item 5 (and item 4's quota_window_blocks leg): one root's distinct
|
|
2063
|
+
# (group key, group digest) pairs stream after three equality columns.
|
|
2064
|
+
("quota_window_blocks",
|
|
2065
|
+
("source_root_key", "source", "orphaned_at", "physical_group_key",
|
|
2066
|
+
"physical_group_digest"),
|
|
2067
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_root_group_pairs "
|
|
2068
|
+
"ON quota_window_blocks(source_root_key ASC, source ASC, orphaned_at ASC, "
|
|
2069
|
+
"physical_group_key ASC, physical_group_digest ASC)"),
|
|
2070
|
+
# Items 6 and 7: the WeekRef reset-event applier and
|
|
2071
|
+
# `in_place_cut_instants` read every event in the chokepoint's order.
|
|
2072
|
+
("week_reset_events", ("effective_reset_at_utc", "id"),
|
|
2073
|
+
"CREATE INDEX IF NOT EXISTS idx_week_reset_events_effective_order "
|
|
2074
|
+
"ON week_reset_events(unixepoch(effective_reset_at_utc) DESC, id DESC)"),
|
|
2075
|
+
# Item 9: `_compute_subscription_weeks` scans representative rows in
|
|
2076
|
+
# (minimum start, date) order and picks and aggregates each date's
|
|
2077
|
+
# qualifying population. The order index's predicate omits the answer's
|
|
2078
|
+
# `week_start_date IS NOT NULL`: the column is NOT NULL, so SQLite drops
|
|
2079
|
+
# that term from the outer scan's WHERE and then cannot prove the partial
|
|
2080
|
+
# index usable ("no query solution" under INDEXED BY, measured). The
|
|
2081
|
+
# indexed population is identical. The pick index keeps the term, because
|
|
2082
|
+
# every lookup through it carries a date equality that implies it.
|
|
2083
|
+
("weekly_usage_snapshots",
|
|
2084
|
+
("week_start_at", "week_end_at", "week_start_date"),
|
|
2085
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_order "
|
|
2086
|
+
"ON weekly_usage_snapshots(week_start_at ASC, week_start_date ASC) "
|
|
2087
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
|
|
2088
|
+
("weekly_usage_snapshots",
|
|
2089
|
+
("week_start_date", "week_start_at", "week_end_at", "id"),
|
|
2090
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_pick "
|
|
2091
|
+
"ON weekly_usage_snapshots(week_start_date ASC, week_start_at ASC, id ASC) "
|
|
2092
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL "
|
|
2093
|
+
"AND week_start_date IS NOT NULL"),
|
|
2094
|
+
)
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _apply_stats_read_indexes(conn: sqlite3.Connection) -> None:
|
|
2098
|
+
"""Create each #901 read index whose table and columns exist (epoch 1017).
|
|
2099
|
+
|
|
2100
|
+
``open_db`` calls this twice: beside the quota projection schema, and
|
|
2101
|
+
again after the stats migration dispatcher, because a legacy store being
|
|
2102
|
+
cut over gains one indexed column (``reset_event_id``, migration 005) only
|
|
2103
|
+
from the dispatcher. Every statement is IF NOT EXISTS, so the second call
|
|
2104
|
+
creates exactly what the first had to skip, and an index is never attempted
|
|
2105
|
+
over a column that is not there yet.
|
|
2106
|
+
"""
|
|
2107
|
+
for table, required, statement in _STATS_READ_INDEX_DDL:
|
|
2108
|
+
columns = {
|
|
2109
|
+
str(row[0]) for row in conn.execute(
|
|
2110
|
+
"SELECT name FROM pragma_table_info(?)", (table,))
|
|
2111
|
+
}
|
|
2112
|
+
if set(required) <= columns:
|
|
2113
|
+
conn.execute(statement)
|
|
2114
|
+
|
|
2115
|
+
|
|
1864
2116
|
def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
1865
2117
|
# ``_target_path`` (internal, keyword-only) builds/opens the stats index at
|
|
1866
2118
|
# an ALTERNATE path instead of the module-global ``DB_PATH`` — the seam
|
|
@@ -2129,6 +2381,10 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2129
2381
|
conn, _STATS_MIGRATIONS, "stats.db"
|
|
2130
2382
|
)
|
|
2131
2383
|
return conn
|
|
2384
|
+
# PR-3: everything below is pending schema, migration or cutover work;
|
|
2385
|
+
# it sorts under FILE, and the writer temp store is selected again
|
|
2386
|
+
# before the connection is returned.
|
|
2387
|
+
_cctally_store.apply_schema_work_temp_store(conn)
|
|
2132
2388
|
_fixups_current = _cctally_store.stats_open_fixups_current(conn)
|
|
2133
2389
|
conn.execute(
|
|
2134
2390
|
"""
|
|
@@ -2854,6 +3110,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2854
3110
|
# without running its handler, so this open-time call is the sole creator).
|
|
2855
3111
|
if not _fixups_current:
|
|
2856
3112
|
_apply_quota_projection_schema(conn)
|
|
3113
|
+
_apply_stats_read_indexes(conn)
|
|
2857
3114
|
|
|
2858
3115
|
# Migration framework dispatcher. Replaces the prior inline gate stack
|
|
2859
3116
|
# (has_blocks + _migration_done) with the framework's _run_pending_-
|
|
@@ -2871,6 +3128,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2871
3128
|
_run_pending_migrations(
|
|
2872
3129
|
conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
|
|
2873
3130
|
)
|
|
3131
|
+
_apply_stats_read_indexes(conn)
|
|
2874
3132
|
|
|
2875
3133
|
# One-time historical backfill of five_hour_blocks (rollup only;
|
|
2876
3134
|
# milestones are forward-only per spec §4.3 / [Write-once milestones]).
|
|
@@ -3089,6 +3347,106 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
3089
3347
|
"cutover_seen INTEGER NOT NULL DEFAULT 0, "
|
|
3090
3348
|
"cutover_account_key TEXT)"
|
|
3091
3349
|
)
|
|
3350
|
+
# #825/#832 epoch 1018 (spec section 8): the bounded, disposable
|
|
3351
|
+
# summary of the complete journal records the admission predicate
|
|
3352
|
+
# EXCLUDED (post-cutover records without the current writer
|
|
3353
|
+
# generation). Ingest extends it inside its cycle transaction and a
|
|
3354
|
+
# rebuild reproduces it from the pinned prefix; `complete_through` is
|
|
3355
|
+
# NULL when the row was started mid-journal (it then describes only
|
|
3356
|
+
# what it saw). Coordinates are JSON `{"segment", "offset"}` text, and
|
|
3357
|
+
# `coordinate_digest` is the order-independent population digest of
|
|
3358
|
+
# every excluded coordinate (`_lib_account_protocol`).
|
|
3359
|
+
conn.execute(
|
|
3360
|
+
"CREATE TABLE IF NOT EXISTS journal_admission_summary ("
|
|
3361
|
+
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
3362
|
+
"cutover_id TEXT, "
|
|
3363
|
+
"scanned_high_water TEXT, "
|
|
3364
|
+
"excluded_total INTEGER NOT NULL DEFAULT 0, "
|
|
3365
|
+
"family_counts_json TEXT NOT NULL DEFAULT '{}', "
|
|
3366
|
+
"first_excluded_coordinate TEXT, "
|
|
3367
|
+
"latest_excluded_coordinate TEXT, "
|
|
3368
|
+
"coordinate_digest TEXT, "
|
|
3369
|
+
"complete_through TEXT, "
|
|
3370
|
+
# The boundary payload this summary classified against, so the
|
|
3371
|
+
# live ingest path resolves admission without scanning the journal.
|
|
3372
|
+
"boundary_json TEXT)"
|
|
3373
|
+
)
|
|
3374
|
+
# #825/#832 fix round A (Codex fix-round-1 Q1): the derivation
|
|
3375
|
+
# frontier. Every admitted derivable input at or before this journal
|
|
3376
|
+
# coordinate has had its decisions journaled by a current binary. Live
|
|
3377
|
+
# ingest advances it with the cursor only while the two are equal; a
|
|
3378
|
+
# rebuild carries it from a trusted predecessor index or sets it after
|
|
3379
|
+
# the shared completion step proved the window. Absent means unknown,
|
|
3380
|
+
# which the completion step reads as "replay from the cutover".
|
|
3381
|
+
# Fix round C (P1-1): the frontier counts CONSUMED INPUTS, and the
|
|
3382
|
+
# decisions a cycle derives from them land beyond it, so the row also
|
|
3383
|
+
# names the output end — the coordinate through which every decision
|
|
3384
|
+
# derived from an input at or before the frontier is journaled, with
|
|
3385
|
+
# nothing between the two derived from a later input. NULL means
|
|
3386
|
+
# nothing is owed beyond the frontier.
|
|
3387
|
+
conn.execute(
|
|
3388
|
+
"CREATE TABLE IF NOT EXISTS journal_derivation_frontier ("
|
|
3389
|
+
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
3390
|
+
"segment TEXT NOT NULL, "
|
|
3391
|
+
"offset INTEGER NOT NULL, "
|
|
3392
|
+
"output_segment TEXT, "
|
|
3393
|
+
"output_offset INTEGER)"
|
|
3394
|
+
)
|
|
3395
|
+
# An index an earlier build of this epoch created lacks the two
|
|
3396
|
+
# columns; additive, so it keeps its frontier row.
|
|
3397
|
+
add_column_if_missing(
|
|
3398
|
+
conn, "journal_derivation_frontier", "output_segment", "TEXT")
|
|
3399
|
+
add_column_if_missing(
|
|
3400
|
+
conn, "journal_derivation_frontier", "output_offset", "INTEGER")
|
|
3401
|
+
# Revision 2 (#946, Codex design-fix-round-3 Q5): the proof
|
|
3402
|
+
# generation. A row written under the old traversal rule (NULL here)
|
|
3403
|
+
# may name a valid endpoint past an input nothing derived, so only
|
|
3404
|
+
# `_cctally_journal.DERIVATION_PROOF_GENERATION` is trusted.
|
|
3405
|
+
add_column_if_missing(
|
|
3406
|
+
conn, "journal_derivation_frontier", "proof_generation", "INTEGER")
|
|
3407
|
+
# #825/#832 fix round A (Codex fix-round-1 Q2): the physical revision
|
|
3408
|
+
# inventory of every EXCLUDED event or correction action the index has
|
|
3409
|
+
# consumed, by logical id. Admission hides these from effective
|
|
3410
|
+
# selection, so without it a current writer would classify the id NEW
|
|
3411
|
+
# and append a revision-0 twin the previous release reads as a
|
|
3412
|
+
# same-revision conflict.
|
|
3413
|
+
conn.execute(
|
|
3414
|
+
"CREATE TABLE IF NOT EXISTS journal_excluded_event_revisions ("
|
|
3415
|
+
"event_id TEXT PRIMARY KEY, "
|
|
3416
|
+
"max_rev INTEGER NOT NULL)"
|
|
3417
|
+
)
|
|
3418
|
+
# #825/#832 epoch 1018 (spec 5.6 item 9): the outcome quarantine. A
|
|
3419
|
+
# deterministic function of journal order that ingest extends inside
|
|
3420
|
+
# its cycle transaction and a rebuild reproduces from the pinned
|
|
3421
|
+
# prefix: one row per uncertainty trigger (an excluded record or a
|
|
3422
|
+
# current `legacy_spool_activity` op, at its physical coordinate), the
|
|
3423
|
+
# whole-prefix envelope per scope ("excluded" applies to every
|
|
3424
|
+
# account), and each account's accounting and control certificates
|
|
3425
|
+
# with the trigger sequence they were granted against. A certificate
|
|
3426
|
+
# granted against an older sequence is void; a missing or
|
|
3427
|
+
# unverifiable row reads as quarantined.
|
|
3428
|
+
conn.execute(
|
|
3429
|
+
"CREATE TABLE IF NOT EXISTS outcome_quarantine_triggers ("
|
|
3430
|
+
"seq INTEGER PRIMARY KEY, "
|
|
3431
|
+
"segment TEXT, "
|
|
3432
|
+
"byte_offset INTEGER, "
|
|
3433
|
+
"kind TEXT NOT NULL, "
|
|
3434
|
+
"reason TEXT NOT NULL, "
|
|
3435
|
+
"at TEXT)"
|
|
3436
|
+
)
|
|
3437
|
+
conn.execute(
|
|
3438
|
+
"CREATE TABLE IF NOT EXISTS outcome_quarantine_envelope ("
|
|
3439
|
+
"scope TEXT PRIMARY KEY, "
|
|
3440
|
+
"envelope_json TEXT NOT NULL)"
|
|
3441
|
+
)
|
|
3442
|
+
conn.execute(
|
|
3443
|
+
"CREATE TABLE IF NOT EXISTS outcome_quarantine_certificates ("
|
|
3444
|
+
"account_key TEXT PRIMARY KEY, "
|
|
3445
|
+
"accounting_certificate_json TEXT, "
|
|
3446
|
+
"accounting_trigger_seq INTEGER, "
|
|
3447
|
+
"control_certificate_json TEXT, "
|
|
3448
|
+
"control_trigger_seq INTEGER)"
|
|
3449
|
+
)
|
|
3092
3450
|
conn.execute(
|
|
3093
3451
|
"CREATE TABLE IF NOT EXISTS journal_selector_batches ("
|
|
3094
3452
|
"batch_id TEXT PRIMARY KEY, "
|
|
@@ -3148,6 +3506,45 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
3148
3506
|
"started_at_utc TEXT NOT NULL, "
|
|
3149
3507
|
"stamped_at_utc TEXT NOT NULL)"
|
|
3150
3508
|
)
|
|
3509
|
+
# #825/#832 Task 4.0 (epoch 1018): a milestone whose snapshot a
|
|
3510
|
+
# credit's stale-replica removal deleted. The removal's band starts at
|
|
3511
|
+
# floor10 of the arming tick, so it also reaches the GENUINE pre-credit
|
|
3512
|
+
# peak tick when that tick shares the arming tick's ten-minute slot —
|
|
3513
|
+
# and that tick is the one that crossed the last weekly and five-hour
|
|
3514
|
+
# milestones. A rebuild folds the suppression (order 30) before the
|
|
3515
|
+
# milestones (order 60), so their journaled `usage_snapshot_ref` named
|
|
3516
|
+
# a row that no longer existed and the NOT NULL row was silently
|
|
3517
|
+
# dropped; the next tick then re-recorded the highest weekly milestone
|
|
3518
|
+
# and its conflict convergence raised on every cycle.
|
|
3519
|
+
#
|
|
3520
|
+
# The tombstone records every journaled snapshot the index deleted —
|
|
3521
|
+
# the live stale-replica removals and the suppression appliers alike,
|
|
3522
|
+
# because a trigger sees every DELETE — so the fold's ONE resolver
|
|
3523
|
+
# (`_cctally_journal._resolve_fold_ref`) can tell "a suppression
|
|
3524
|
+
# removed it" from a genuinely unresolvable reference. The trigger's
|
|
3525
|
+
# second half points every milestone that referenced the deleted row at
|
|
3526
|
+
# the 0 "no snapshot row" sentinel, which is what the rebuild resolves
|
|
3527
|
+
# such a reference to, so the live index and a rebuild agree. Both
|
|
3528
|
+
# milestone readers already treat 0 as "no row": the five-hour
|
|
3529
|
+
# milestone read LEFT JOINs (and so publishes the effective weekly
|
|
3530
|
+
# value as unavailable, as it already did for the dangling rowid), and
|
|
3531
|
+
# the observation-gap classifier never continues a run through 0.
|
|
3532
|
+
conn.execute(
|
|
3533
|
+
"CREATE TABLE IF NOT EXISTS usage_snapshot_suppressions ("
|
|
3534
|
+
"journal_id TEXT PRIMARY KEY)"
|
|
3535
|
+
)
|
|
3536
|
+
conn.execute(
|
|
3537
|
+
"CREATE TRIGGER IF NOT EXISTS trg_usage_snapshot_suppression "
|
|
3538
|
+
"AFTER DELETE ON weekly_usage_snapshots "
|
|
3539
|
+
"BEGIN "
|
|
3540
|
+
"INSERT OR IGNORE INTO usage_snapshot_suppressions (journal_id) "
|
|
3541
|
+
"SELECT OLD.journal_id WHERE OLD.journal_id IS NOT NULL; "
|
|
3542
|
+
"UPDATE percent_milestones SET usage_snapshot_id = 0 "
|
|
3543
|
+
"WHERE usage_snapshot_id = OLD.id; "
|
|
3544
|
+
"UPDATE five_hour_milestones SET usage_snapshot_id = 0 "
|
|
3545
|
+
"WHERE usage_snapshot_id = OLD.id; "
|
|
3546
|
+
"END"
|
|
3547
|
+
)
|
|
3151
3548
|
|
|
3152
3549
|
# §6.2 backfill gate (Task 8): stamp the one-shot marker AFTER the three
|
|
3153
3550
|
# open-time backfills ran, so the next open skips them (and their probes)
|
|
@@ -3186,6 +3583,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
3186
3583
|
except Exception:
|
|
3187
3584
|
pass
|
|
3188
3585
|
raise
|
|
3586
|
+
_cctally_store.apply_writer_temp_store(conn)
|
|
3189
3587
|
return conn
|
|
3190
3588
|
|
|
3191
3589
|
|