cctally 1.113.0 → 1.114.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +4 -6
- package/bin/_cctally_cache.py +1683 -485
- package/bin/_cctally_core.py +223 -1
- package/bin/_cctally_dashboard.py +96 -18
- package/bin/_cctally_dashboard_perf.py +128 -9
- package/bin/_cctally_dashboard_sources.py +267 -204
- package/bin/_cctally_db.py +861 -18
- package/bin/_cctally_diagnosis_sources.py +22 -4
- package/bin/_cctally_doctor.py +136 -18
- package/bin/_cctally_five_hour.py +2 -2
- package/bin/_cctally_forecast.py +27 -6
- package/bin/_cctally_journal.py +67 -5
- package/bin/_cctally_milestone_history.py +82 -9
- package/bin/_cctally_percent_breakdown.py +3 -2
- package/bin/_cctally_quota.py +201 -33
- package/bin/_cctally_record.py +14 -0
- package/bin/_cctally_rederive.py +23 -9
- package/bin/_cctally_source_analytics.py +60 -15
- package/bin/_cctally_statusline.py +17 -0
- package/bin/_cctally_store.py +80 -5
- package/bin/_cctally_tui.py +4 -3
- package/bin/_cctally_weekrefs.py +128 -21
- package/bin/_lib_cache_report.py +35 -6
- package/bin/_lib_codex_conversation.py +32 -0
- package/bin/_lib_codex_conversation_query.py +164 -30
- package/bin/_lib_conversation_query.py +95 -56
- package/bin/_lib_conversation_retention.py +1605 -640
- package/bin/_lib_cost_provenance.py +53 -2
- package/bin/_lib_doctor.py +263 -69
- package/bin/_lib_fmt.py +11 -0
- package/bin/_lib_merge_gate.py +469 -51
- package/bin/_lib_perf.py +27 -1
- package/bin/_lib_pricing.py +193 -8
- package/bin/_lib_pricing_check.py +20 -4
- package/bin/_lib_reclaim_planner.py +792 -0
- package/bin/_lib_rederive.py +225 -77
- package/bin/_lib_snapshot_cache.py +21 -0
- package/bin/_lib_sqlite_close.py +81 -0
- package/bin/_lib_subscription_weeks.py +100 -17
- package/bin/_lib_tick_stats.py +217 -17
- package/bin/_lib_wal_checkpoint.py +1077 -0
- package/bin/_lib_write_budget.py +483 -0
- package/bin/_lib_write_io.py +703 -0
- package/bin/cctally +8 -0
- package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
- package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
- package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
- package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
- package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
- package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
- package/dashboard/static/assets/index-BKITk1CT.css +1 -0
- package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
- package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +6 -1
- package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
- package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
package/bin/_cctally_core.py
CHANGED
|
@@ -473,7 +473,21 @@ _init_paths_from_env()
|
|
|
473
473
|
# tick must carry that earlier evidence into the journal, even after this
|
|
474
474
|
# binary learns a new model card. A pre-existing open projection that was
|
|
475
475
|
# rebuilt without the marker remains unproven until a new tick recomputes it.
|
|
476
|
-
|
|
476
|
+
# 1016 -> 1017 (#901 §5.3, T3): read indexes only. Every stats digest relation
|
|
477
|
+
# (`_lib_dashboard_sources._CODEX_STATS_DIGEST_RELATIONS` and
|
|
478
|
+
# `_CLAUDE_STATS_DIGEST_RELATIONS`) sorted its whole relation through a temp
|
|
479
|
+
# b-tree on every dashboard build, and `_fetch_current_week_snapshots` grouped
|
|
480
|
+
# every boundary-aware and legacy-date snapshot the same way; each now streams
|
|
481
|
+
# through an index in its own order (`_STATS_READ_INDEX_DDL`). No table or
|
|
482
|
+
# column changes and no digest byte moves. The same epoch carries Amendment 1's
|
|
483
|
+
# history-read indexes (items 2-7 and 9, `c901-design-2`, with item 2's second
|
|
484
|
+
# single-root index from Amendment 1c, `c901-design-3`): every full dashboard
|
|
485
|
+
# build enumerated weekly blocks, milestone weeks, root keys, group pairs,
|
|
486
|
+
# reset events and subscription anchors through temp b-trees. The mechanical
|
|
487
|
+
# reason this is a bump and not a migration is unchanged: the registry is
|
|
488
|
+
# frozen at 13, and an epoch-current open returns before any schema work, so a
|
|
489
|
+
# schema helper alone would never run on an upgraded install.
|
|
490
|
+
STATS_INDEX_EPOCH = 1017
|
|
477
491
|
LEGACY_STATS_HEAD = 13
|
|
478
492
|
|
|
479
493
|
#: #496 S1 F1. A NEW branch, for a state that cannot occur before the
|
|
@@ -1861,6 +1875,207 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
|
|
|
1861
1875
|
"next_evaluation_by_root_json", "TEXT NOT NULL DEFAULT '{}'")
|
|
1862
1876
|
|
|
1863
1877
|
|
|
1878
|
+
#: #901 §5.3 (T3), epoch 1017: (table, columns the index reads, DDL). Each
|
|
1879
|
+
#: index's leading columns are the statement's equality filter (when it has
|
|
1880
|
+
#: one) followed by its ORDER BY / GROUP BY list, so the statement streams off
|
|
1881
|
+
#: the index with no sorter; the digest indexes are covering, because every
|
|
1882
|
+
#: digest relation selects exactly its ordering columns. SQLite stores each
|
|
1883
|
+
#: statement verbatim in `sqlite_schema`, which `_REBUILD_SCHEMA_FINGERPRINT`
|
|
1884
|
+
#: hashes: change the text and the fingerprint must move with it.
|
|
1885
|
+
_STATS_READ_INDEX_DDL: "tuple[tuple[str, tuple[str, ...], str], ...]" = (
|
|
1886
|
+
("quota_projection_state", ("source_root_key", "physical_signature"),
|
|
1887
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_projection_state_digest "
|
|
1888
|
+
"ON quota_projection_state(source_root_key, physical_signature)"),
|
|
1889
|
+
("quota_window_blocks",
|
|
1890
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1891
|
+
"window_minutes", "limit_id", "limit_name", "resets_at_utc",
|
|
1892
|
+
"nominal_start_at_utc", "first_observed_at_utc", "last_observed_at_utc",
|
|
1893
|
+
"first_percent", "current_percent", "orphaned_at"),
|
|
1894
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_digest "
|
|
1895
|
+
"ON quota_window_blocks(source, source_root_key, logical_limit_key, "
|
|
1896
|
+
"observed_slot, window_minutes, limit_id, limit_name, resets_at_utc, "
|
|
1897
|
+
"nominal_start_at_utc, first_observed_at_utc, last_observed_at_utc, "
|
|
1898
|
+
"first_percent, current_percent, orphaned_at) WHERE source='codex'"),
|
|
1899
|
+
("quota_percent_milestones",
|
|
1900
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1901
|
+
"window_minutes", "resets_at_utc", "percent_threshold",
|
|
1902
|
+
"captured_at_utc", "high_water_percent", "orphaned_at"),
|
|
1903
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_milestones_digest "
|
|
1904
|
+
"ON quota_percent_milestones(source, source_root_key, logical_limit_key, "
|
|
1905
|
+
"observed_slot, window_minutes, resets_at_utc, percent_threshold, "
|
|
1906
|
+
"captured_at_utc, high_water_percent, orphaned_at) WHERE source='codex'"),
|
|
1907
|
+
("quota_threshold_events",
|
|
1908
|
+
("source", "source_root_key", "logical_limit_key", "observed_slot",
|
|
1909
|
+
"window_minutes", "resets_at_utc", "threshold", "qualifying_kind",
|
|
1910
|
+
"qualifying_percent", "projected_percent", "severity", "created_at_utc",
|
|
1911
|
+
"disposition", "alerted_at", "suppressed_at", "orphaned_at"),
|
|
1912
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_digest "
|
|
1913
|
+
"ON quota_threshold_events(source, source_root_key, logical_limit_key, "
|
|
1914
|
+
"observed_slot, window_minutes, resets_at_utc, threshold, "
|
|
1915
|
+
"qualifying_kind, qualifying_percent, projected_percent, severity, "
|
|
1916
|
+
"created_at_utc, disposition, alerted_at, suppressed_at, orphaned_at) "
|
|
1917
|
+
"WHERE source='codex'"),
|
|
1918
|
+
("budget_milestones",
|
|
1919
|
+
("vendor", "period_start_at", "period", "threshold", "budget_usd",
|
|
1920
|
+
"spent_usd", "consumption_pct", "crossed_at_utc", "alerted_at"),
|
|
1921
|
+
"CREATE INDEX IF NOT EXISTS idx_budget_milestones_codex_digest "
|
|
1922
|
+
"ON budget_milestones(vendor, period_start_at, period, threshold, "
|
|
1923
|
+
"budget_usd, spent_usd, consumption_pct, crossed_at_utc, alerted_at) "
|
|
1924
|
+
"WHERE vendor='codex'"),
|
|
1925
|
+
("projected_milestones",
|
|
1926
|
+
("metric", "week_start_at", "period", "threshold", "projected_value",
|
|
1927
|
+
"denominator", "crossed_at_utc", "alerted_at"),
|
|
1928
|
+
"CREATE INDEX IF NOT EXISTS idx_projected_milestones_codex_digest "
|
|
1929
|
+
"ON projected_milestones(metric, week_start_at, period, threshold, "
|
|
1930
|
+
"projected_value, denominator, crossed_at_utc, alerted_at) "
|
|
1931
|
+
"WHERE metric='codex_budget_usd'"),
|
|
1932
|
+
("percent_milestones",
|
|
1933
|
+
("week_start_date", "percent_threshold", "reset_event_id", "account_key",
|
|
1934
|
+
"captured_at_utc", "cumulative_cost_usd", "alerted_at"),
|
|
1935
|
+
"CREATE INDEX IF NOT EXISTS idx_percent_milestones_alert_digest "
|
|
1936
|
+
"ON percent_milestones(week_start_date, percent_threshold, "
|
|
1937
|
+
"reset_event_id, account_key, captured_at_utc, cumulative_cost_usd, "
|
|
1938
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
1939
|
+
("five_hour_milestones",
|
|
1940
|
+
("five_hour_window_key", "percent_threshold", "reset_event_id",
|
|
1941
|
+
"account_key", "captured_at_utc", "block_cost_usd", "alerted_at"),
|
|
1942
|
+
"CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_alert_digest "
|
|
1943
|
+
"ON five_hour_milestones(five_hour_window_key, percent_threshold, "
|
|
1944
|
+
"reset_event_id, account_key, captured_at_utc, block_cost_usd, "
|
|
1945
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
1946
|
+
("budget_milestones",
|
|
1947
|
+
("vendor", "period_start_at", "period", "threshold", "account_key",
|
|
1948
|
+
"budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
|
|
1949
|
+
"alerted_at"),
|
|
1950
|
+
"CREATE INDEX IF NOT EXISTS idx_budget_milestones_alert_digest "
|
|
1951
|
+
"ON budget_milestones(vendor, period_start_at, period, threshold, "
|
|
1952
|
+
"account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
|
|
1953
|
+
"alerted_at) WHERE vendor <> 'codex' AND alerted_at IS NOT NULL"),
|
|
1954
|
+
("projected_milestones",
|
|
1955
|
+
("week_start_at", "period", "metric", "threshold", "account_key",
|
|
1956
|
+
"projected_value", "denominator", "crossed_at_utc", "alerted_at"),
|
|
1957
|
+
"CREATE INDEX IF NOT EXISTS idx_projected_milestones_alert_digest "
|
|
1958
|
+
"ON projected_milestones(week_start_at, period, metric, threshold, "
|
|
1959
|
+
"account_key, projected_value, denominator, crossed_at_utc, alerted_at) "
|
|
1960
|
+
"WHERE metric <> 'codex_budget_usd' AND alerted_at IS NOT NULL"),
|
|
1961
|
+
("project_budget_milestones",
|
|
1962
|
+
("week_start_at", "project_key", "threshold", "account_key",
|
|
1963
|
+
"budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
|
|
1964
|
+
"alerted_at"),
|
|
1965
|
+
"CREATE INDEX IF NOT EXISTS idx_project_budget_milestones_alert_digest "
|
|
1966
|
+
"ON project_budget_milestones(week_start_at, project_key, threshold, "
|
|
1967
|
+
"account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
|
|
1968
|
+
"alerted_at) WHERE alerted_at IS NOT NULL"),
|
|
1969
|
+
("weekly_usage_snapshots",
|
|
1970
|
+
("week_start_at", "week_end_at", "week_start_date", "account_key",
|
|
1971
|
+
"weekly_observation_held", "captured_at_utc"),
|
|
1972
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_week_boundary_group "
|
|
1973
|
+
"ON weekly_usage_snapshots(week_start_at, week_end_at, week_start_date, "
|
|
1974
|
+
"account_key, weekly_observation_held, captured_at_utc) "
|
|
1975
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
|
|
1976
|
+
("weekly_usage_snapshots",
|
|
1977
|
+
("week_start_date", "week_end_date", "account_key",
|
|
1978
|
+
"weekly_observation_held", "captured_at_utc"),
|
|
1979
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_week_date_group "
|
|
1980
|
+
"ON weekly_usage_snapshots(week_start_date, week_end_date, account_key, "
|
|
1981
|
+
"weekly_observation_held, captured_at_utc)"),
|
|
1982
|
+
# Amendment 1 (binding Codex answer `c901-design-2`): the recurring history
|
|
1983
|
+
# reads the #901 plan guard escalated. Item 2: `_load_codex_cycles` orders
|
|
1984
|
+
# every weekly block of the active roots by reset; the partial predicate
|
|
1985
|
+
# deliberately keeps orphaned rows so `include_orphaned=True` streams too.
|
|
1986
|
+
# Amendment 1b widened it past the reset second: `_canonicalize_codex_
|
|
1987
|
+
# cluster` keeps the first maximal member, so the order of rows tied on one
|
|
1988
|
+
# second is observable, and the trailing columns make it explicit. The
|
|
1989
|
+
# table's UNIQUE constraint makes them unique within the population.
|
|
1990
|
+
# Amendment 1c (`c901-design-3`): the legacy plan fed its sorter in two
|
|
1991
|
+
# different orders. A read of two or more roots scanned
|
|
1992
|
+
# `idx_quota_blocks_active` (ties by root, account, orphaned_at, limit,
|
|
1993
|
+
# slot, reset spelling); a read of exactly one root seeked the UNIQUE
|
|
1994
|
+
# autoindex (the same order without orphaned_at). The first index below
|
|
1995
|
+
# serves the multi-root read, the second the single-root read, each in its
|
|
1996
|
+
# legacy tie order.
|
|
1997
|
+
("quota_window_blocks",
|
|
1998
|
+
("source", "window_minutes", "resets_at_utc", "source_root_key",
|
|
1999
|
+
"account_key", "orphaned_at", "logical_limit_key", "observed_slot"),
|
|
2000
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_reset_order "
|
|
2001
|
+
"ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
|
|
2002
|
+
"source_root_key ASC, account_key ASC, orphaned_at ASC, "
|
|
2003
|
+
"logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
|
|
2004
|
+
"WHERE source='codex' AND window_minutes=10080"),
|
|
2005
|
+
("quota_window_blocks",
|
|
2006
|
+
("source", "window_minutes", "resets_at_utc", "source_root_key",
|
|
2007
|
+
"account_key", "logical_limit_key", "observed_slot"),
|
|
2008
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_single_root_order "
|
|
2009
|
+
"ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
|
|
2010
|
+
"source_root_key ASC, account_key ASC, "
|
|
2011
|
+
"logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
|
|
2012
|
+
"WHERE source='codex' AND window_minutes=10080"),
|
|
2013
|
+
# Item 3: `_navigable_claude_refs` de-duplicates every milestone week; a
|
|
2014
|
+
# full index, because the alert digest index cannot see unalerted rows.
|
|
2015
|
+
("percent_milestones", ("week_start_date",),
|
|
2016
|
+
"CREATE INDEX IF NOT EXISTS idx_percent_milestones_week_date "
|
|
2017
|
+
"ON percent_milestones(week_start_date ASC)"),
|
|
2018
|
+
# Item 4: `_historic_root_keys` reads the complete relations.
|
|
2019
|
+
("quota_percent_milestones", ("source_root_key",),
|
|
2020
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_milestones_root "
|
|
2021
|
+
"ON quota_percent_milestones(source_root_key ASC)"),
|
|
2022
|
+
("quota_threshold_events", ("source_root_key",),
|
|
2023
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_root "
|
|
2024
|
+
"ON quota_threshold_events(source_root_key ASC)"),
|
|
2025
|
+
# Item 5 (and item 4's quota_window_blocks leg): one root's distinct
|
|
2026
|
+
# (group key, group digest) pairs stream after three equality columns.
|
|
2027
|
+
("quota_window_blocks",
|
|
2028
|
+
("source_root_key", "source", "orphaned_at", "physical_group_key",
|
|
2029
|
+
"physical_group_digest"),
|
|
2030
|
+
"CREATE INDEX IF NOT EXISTS idx_quota_blocks_root_group_pairs "
|
|
2031
|
+
"ON quota_window_blocks(source_root_key ASC, source ASC, orphaned_at ASC, "
|
|
2032
|
+
"physical_group_key ASC, physical_group_digest ASC)"),
|
|
2033
|
+
# Items 6 and 7: the WeekRef reset-event applier and
|
|
2034
|
+
# `in_place_cut_instants` read every event in the chokepoint's order.
|
|
2035
|
+
("week_reset_events", ("effective_reset_at_utc", "id"),
|
|
2036
|
+
"CREATE INDEX IF NOT EXISTS idx_week_reset_events_effective_order "
|
|
2037
|
+
"ON week_reset_events(unixepoch(effective_reset_at_utc) DESC, id DESC)"),
|
|
2038
|
+
# Item 9: `_compute_subscription_weeks` scans representative rows in
|
|
2039
|
+
# (minimum start, date) order and picks and aggregates each date's
|
|
2040
|
+
# qualifying population. The order index's predicate omits the answer's
|
|
2041
|
+
# `week_start_date IS NOT NULL`: the column is NOT NULL, so SQLite drops
|
|
2042
|
+
# that term from the outer scan's WHERE and then cannot prove the partial
|
|
2043
|
+
# index usable ("no query solution" under INDEXED BY, measured). The
|
|
2044
|
+
# indexed population is identical. The pick index keeps the term, because
|
|
2045
|
+
# every lookup through it carries a date equality that implies it.
|
|
2046
|
+
("weekly_usage_snapshots",
|
|
2047
|
+
("week_start_at", "week_end_at", "week_start_date"),
|
|
2048
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_order "
|
|
2049
|
+
"ON weekly_usage_snapshots(week_start_at ASC, week_start_date ASC) "
|
|
2050
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
|
|
2051
|
+
("weekly_usage_snapshots",
|
|
2052
|
+
("week_start_date", "week_start_at", "week_end_at", "id"),
|
|
2053
|
+
"CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_pick "
|
|
2054
|
+
"ON weekly_usage_snapshots(week_start_date ASC, week_start_at ASC, id ASC) "
|
|
2055
|
+
"WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL "
|
|
2056
|
+
"AND week_start_date IS NOT NULL"),
|
|
2057
|
+
)
|
|
2058
|
+
|
|
2059
|
+
|
|
2060
|
+
def _apply_stats_read_indexes(conn: sqlite3.Connection) -> None:
|
|
2061
|
+
"""Create each #901 read index whose table and columns exist (epoch 1017).
|
|
2062
|
+
|
|
2063
|
+
``open_db`` calls this twice: beside the quota projection schema, and
|
|
2064
|
+
again after the stats migration dispatcher, because a legacy store being
|
|
2065
|
+
cut over gains one indexed column (``reset_event_id``, migration 005) only
|
|
2066
|
+
from the dispatcher. Every statement is IF NOT EXISTS, so the second call
|
|
2067
|
+
creates exactly what the first had to skip, and an index is never attempted
|
|
2068
|
+
over a column that is not there yet.
|
|
2069
|
+
"""
|
|
2070
|
+
for table, required, statement in _STATS_READ_INDEX_DDL:
|
|
2071
|
+
columns = {
|
|
2072
|
+
str(row[0]) for row in conn.execute(
|
|
2073
|
+
"SELECT name FROM pragma_table_info(?)", (table,))
|
|
2074
|
+
}
|
|
2075
|
+
if set(required) <= columns:
|
|
2076
|
+
conn.execute(statement)
|
|
2077
|
+
|
|
2078
|
+
|
|
1864
2079
|
def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
1865
2080
|
# ``_target_path`` (internal, keyword-only) builds/opens the stats index at
|
|
1866
2081
|
# an ALTERNATE path instead of the module-global ``DB_PATH`` — the seam
|
|
@@ -2129,6 +2344,10 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2129
2344
|
conn, _STATS_MIGRATIONS, "stats.db"
|
|
2130
2345
|
)
|
|
2131
2346
|
return conn
|
|
2347
|
+
# PR-3: everything below is pending schema, migration or cutover work;
|
|
2348
|
+
# it sorts under FILE, and the writer temp store is selected again
|
|
2349
|
+
# before the connection is returned.
|
|
2350
|
+
_cctally_store.apply_schema_work_temp_store(conn)
|
|
2132
2351
|
_fixups_current = _cctally_store.stats_open_fixups_current(conn)
|
|
2133
2352
|
conn.execute(
|
|
2134
2353
|
"""
|
|
@@ -2854,6 +3073,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2854
3073
|
# without running its handler, so this open-time call is the sole creator).
|
|
2855
3074
|
if not _fixups_current:
|
|
2856
3075
|
_apply_quota_projection_schema(conn)
|
|
3076
|
+
_apply_stats_read_indexes(conn)
|
|
2857
3077
|
|
|
2858
3078
|
# Migration framework dispatcher. Replaces the prior inline gate stack
|
|
2859
3079
|
# (has_blocks + _migration_done) with the framework's _run_pending_-
|
|
@@ -2871,6 +3091,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2871
3091
|
_run_pending_migrations(
|
|
2872
3092
|
conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
|
|
2873
3093
|
)
|
|
3094
|
+
_apply_stats_read_indexes(conn)
|
|
2874
3095
|
|
|
2875
3096
|
# One-time historical backfill of five_hour_blocks (rollup only;
|
|
2876
3097
|
# milestones are forward-only per spec §4.3 / [Write-once milestones]).
|
|
@@ -3186,6 +3407,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
3186
3407
|
except Exception:
|
|
3187
3408
|
pass
|
|
3188
3409
|
raise
|
|
3410
|
+
_cctally_store.apply_writer_temp_store(conn)
|
|
3189
3411
|
return conn
|
|
3190
3412
|
|
|
3191
3413
|
|
|
@@ -764,6 +764,7 @@ from _lib_snapshot_cache import (
|
|
|
764
764
|
_reset_sig as _snapshot_reset_sig,
|
|
765
765
|
)
|
|
766
766
|
import _lib_tick_stats
|
|
767
|
+
import _lib_write_io
|
|
767
768
|
|
|
768
769
|
|
|
769
770
|
# === F22a: count the silent Group A cache-open failures (#583 S1 §1.6) =====
|
|
@@ -2855,11 +2856,13 @@ def _conversation_sync_loop(
|
|
|
2855
2856
|
while not stop.is_set():
|
|
2856
2857
|
t0 = monotonic()
|
|
2857
2858
|
cpu0 = thread_time_ns()
|
|
2859
|
+
written = _lib_write_io.begin_interval("conversation")
|
|
2858
2860
|
try:
|
|
2859
2861
|
status = run_iteration() or "ok"
|
|
2860
2862
|
except Exception: # noqa: BLE001 — the worker must outlive one bad pass
|
|
2861
2863
|
_log_sync_iteration_failure()
|
|
2862
2864
|
status = "error"
|
|
2865
|
+
observation = _lib_write_io.end_interval(written)
|
|
2863
2866
|
work = max(0.0, monotonic() - t0)
|
|
2864
2867
|
cpu_ns = max(0, thread_time_ns() - cpu0)
|
|
2865
2868
|
seq += 1
|
|
@@ -2882,6 +2885,12 @@ def _conversation_sync_loop(
|
|
|
2882
2885
|
"codex", "not_observed"),
|
|
2883
2886
|
claude_files=getattr(status, "files", {}).get("claude", 0),
|
|
2884
2887
|
codex_files=getattr(status, "files", {}).get("codex", 0),
|
|
2888
|
+
# #901 §5.5: the pass's process write delta. It includes the
|
|
2889
|
+
# main loop's concurrent work, which `write_overlap` flags; it
|
|
2890
|
+
# is never summed with a tick's.
|
|
2891
|
+
process_write_bytes=observation.process_write_bytes,
|
|
2892
|
+
write_status=observation.status,
|
|
2893
|
+
write_overlap=observation.overlap,
|
|
2885
2894
|
)
|
|
2886
2895
|
deadline = _conversation_next_deadline(t0, interval, work)
|
|
2887
2896
|
remaining = deadline - monotonic()
|
|
@@ -6894,6 +6903,38 @@ def _assemble_projects_via_cache(
|
|
|
6894
6903
|
return buckets, total_cost_by_week, key_by_bucket
|
|
6895
6904
|
|
|
6896
6905
|
|
|
6906
|
+
def _projects_week_usage_rows(
|
|
6907
|
+
conn: "sqlite3.Connection", since_date: str, end_date: str,
|
|
6908
|
+
) -> list:
|
|
6909
|
+
"""The anchored, non-NULL weekly readings of the projects window, oldest
|
|
6910
|
+
capture first: ``(week_start_date, week_start_at, weekly_percent,
|
|
6911
|
+
captured_at_utc, id)`` for every week date in ``[since_date, end_date)``.
|
|
6912
|
+
|
|
6913
|
+
The window holds ``weeks_back`` weeks of status-line readings, which is
|
|
6914
|
+
not a bound on a sorter, so the rows are ordered here rather than by an
|
|
6915
|
+
``ORDER BY`` that built a temp b-tree over all of them on every build
|
|
6916
|
+
(#901 Q11). ``(captured_at_utc, id)`` is a total order — ``id`` is the
|
|
6917
|
+
primary key and ``captured_at_utc`` is NOT NULL TEXT, which Python
|
|
6918
|
+
compares in the same code-point order as SQLite's BINARY collation — so
|
|
6919
|
+
the result is exactly the former ``ORDER BY captured_at_utc ASC, id ASC``.
|
|
6920
|
+
"""
|
|
6921
|
+
rows = conn.execute(
|
|
6922
|
+
" SELECT week_start_date, week_start_at, weekly_percent,"
|
|
6923
|
+
" captured_at_utc, id"
|
|
6924
|
+
" FROM weekly_usage_snapshots"
|
|
6925
|
+
# Held rows excluded: these rows feed `latest_usage_by_segment`, which
|
|
6926
|
+
# is a weekly read, and the pre-fetch must select the same population
|
|
6927
|
+
# the reducer would select for itself (#769 S11, #824).
|
|
6928
|
+
" WHERE week_start_date >= ? AND week_start_date < ?"
|
|
6929
|
+
" AND date(week_start_date) IS NOT NULL"
|
|
6930
|
+
" AND weekly_percent IS NOT NULL"
|
|
6931
|
+
+ _cctally_core.weekly_held_exclusion(conn),
|
|
6932
|
+
(since_date, end_date),
|
|
6933
|
+
).fetchall()
|
|
6934
|
+
rows.sort(key=lambda row: (row[3], row[4]))
|
|
6935
|
+
return rows
|
|
6936
|
+
|
|
6937
|
+
|
|
6897
6938
|
def _build_projects_envelope(
|
|
6898
6939
|
conn: "sqlite3.Connection",
|
|
6899
6940
|
*,
|
|
@@ -7165,24 +7206,8 @@ def _build_projects_envelope(
|
|
|
7165
7206
|
# valid rendered-week row (#620 S1 A3).
|
|
7166
7207
|
weekly_pct_by_week: dict[dt.datetime, float] = {}
|
|
7167
7208
|
try:
|
|
7168
|
-
|
|
7169
|
-
|
|
7170
|
-
" captured_at_utc, id"
|
|
7171
|
-
" FROM weekly_usage_snapshots"
|
|
7172
|
-
# Held rows excluded: these rows feed `latest_usage_by_segment`,
|
|
7173
|
-
# which is a weekly read, and the pre-fetch must select the same
|
|
7174
|
-
# population the reducer would select for itself (#769 S11, #824).
|
|
7175
|
-
" WHERE week_start_date >= ? AND week_start_date < ?"
|
|
7176
|
-
" AND date(week_start_date) IS NOT NULL"
|
|
7177
|
-
" AND weekly_percent IS NOT NULL"
|
|
7178
|
-
+ _cctally_core.weekly_held_exclusion(conn) +
|
|
7179
|
-
" ORDER BY captured_at_utc ASC, id ASC",
|
|
7180
|
-
(
|
|
7181
|
-
since_dt.date().isoformat(),
|
|
7182
|
-
cw_end.date().isoformat(),
|
|
7183
|
-
),
|
|
7184
|
-
)
|
|
7185
|
-
rows = cur.fetchall()
|
|
7209
|
+
rows = _projects_week_usage_rows(
|
|
7210
|
+
conn, since_dt.date().isoformat(), cw_end.date().isoformat())
|
|
7186
7211
|
except sqlite3.OperationalError:
|
|
7187
7212
|
# No weekly_usage_snapshots table — leaves attributed_pct = None
|
|
7188
7213
|
# throughout (acceptable per spec §2.7).
|
|
@@ -8086,6 +8111,15 @@ class _DiagnosisSelectorError(Exception):
|
|
|
8086
8111
|
|
|
8087
8112
|
|
|
8088
8113
|
# ── /api/debug/backend on-demand cache-state helpers (issue #276, Session A) ──
|
|
8114
|
+
|
|
8115
|
+
def _debug_write_io() -> dict:
|
|
8116
|
+
"""`writeIo` for the debug surfaces (#901 §5.5). A telemetry fault never
|
|
8117
|
+
takes the diagnostic down, and is never reported as a zero."""
|
|
8118
|
+
try:
|
|
8119
|
+
return _lib_write_io.write_io_payload()
|
|
8120
|
+
except Exception: # noqa: BLE001 — diagnostics fail closed
|
|
8121
|
+
return {"status": "unavailable", "reason": "counter_error"}
|
|
8122
|
+
|
|
8089
8123
|
# All read-only, cheap, and privacy-safe: they leak ONLY row counts, signature
|
|
8090
8124
|
# legs (ints/tuples), pending-flag names, and the tool version — never prompt /
|
|
8091
8125
|
# prose / paths. Computed on demand so cache_state is available even with
|
|
@@ -8494,6 +8528,7 @@ _GET_ROUTES = (
|
|
|
8494
8528
|
("exact", "/api/diagnosis", "_handle_get_diagnosis",
|
|
8495
8529
|
("scope", "endpoint.diagnosis"), False),
|
|
8496
8530
|
("exact", "/api/debug/backend", "_handle_get_debug_backend", None, False),
|
|
8531
|
+
("exact", "/api/debug/backend/write-io", "_handle_get_debug_backend_write_io", None, False),
|
|
8497
8532
|
("exact", "/api/conversations/facets", "_handle_get_conversations_facets",
|
|
8498
8533
|
("scope", "endpoint.conversations_facets"), False),
|
|
8499
8534
|
("exact", "/api/conversations", "_handle_get_conversations",
|
|
@@ -9224,11 +9259,26 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
9224
9259
|
# stats failure reported as a cache one sends the user to
|
|
9225
9260
|
# `cctally cache-sync --rebuild`.
|
|
9226
9261
|
"stats_faults": stats_faults,
|
|
9262
|
+
# #901 §5.5: kernel-accounted process writes, the trailing
|
|
9263
|
+
# five-minute statistic and its budget verdict. Additive camelCase.
|
|
9264
|
+
"writeIo": _debug_write_io(),
|
|
9227
9265
|
}
|
|
9228
9266
|
if body["phases"] is None:
|
|
9229
9267
|
body["note"] = "tracing_disabled"
|
|
9230
9268
|
self._respond_json(200, body)
|
|
9231
9269
|
|
|
9270
|
+
def _handle_get_debug_backend_write_io(self) -> None:
|
|
9271
|
+
"""GET ``/api/debug/backend/write-io`` — the ``writeIo`` object alone
|
|
9272
|
+
(#901 §5.6): what ``cctally doctor`` reads from a running dashboard
|
|
9273
|
+
inside its bounded timeout, without ``/api/debug/backend``'s
|
|
9274
|
+
on-demand cache-table counts. Same gates, same order: the bearer
|
|
9275
|
+
(``_require_api_auth``, before dispatch), then the loopback TCP peer
|
|
9276
|
+
and the IP-literal ``Host``. No bypass exists."""
|
|
9277
|
+
if not self._require_debug_backend_allowed():
|
|
9278
|
+
return
|
|
9279
|
+
self._respond_json(200, {"schemaVersion": 1,
|
|
9280
|
+
"writeIo": _debug_write_io()})
|
|
9281
|
+
|
|
9232
9282
|
def _handle_post_debug_backend_trace(self) -> None:
|
|
9233
9283
|
"""POST ``/api/debug/backend/trace`` — arm the deep phase trace (§3.2).
|
|
9234
9284
|
|
|
@@ -12485,6 +12535,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
12485
12535
|
**_make_sync_loop_collaborators(ref=ref, hub=hub),
|
|
12486
12536
|
)
|
|
12487
12537
|
|
|
12538
|
+
# #901 §5.6: this process's write-telemetry identity, registered before
|
|
12539
|
+
# the first build so the in-process doctor reads its own snapshot from the
|
|
12540
|
+
# first tick; the bound port replaces the requested one once known.
|
|
12541
|
+
write_instance_id = _lib_write_io.new_instance_id()
|
|
12542
|
+
_lib_write_io.register_local_instance(
|
|
12543
|
+
instance_id=write_instance_id, host=args.host,
|
|
12544
|
+
port=_resolve_dashboard_port(args.port))
|
|
12488
12545
|
sync_thread = (
|
|
12489
12546
|
None if args.no_sync
|
|
12490
12547
|
else _DashboardSyncThread(
|
|
@@ -12566,6 +12623,20 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
12566
12623
|
http_thread = threading.Thread(target=srv.serve_forever, daemon=True,
|
|
12567
12624
|
name="dashboard-http")
|
|
12568
12625
|
http_thread.start()
|
|
12626
|
+
# The owner-readable descriptor `cctally doctor` discovers this instance
|
|
12627
|
+
# through: written once now, removed at clean shutdown. A crash leaves it
|
|
12628
|
+
# for the next startup to remove; doctor ignores a dead or reused pid's.
|
|
12629
|
+
_lib_write_io.register_local_instance(
|
|
12630
|
+
instance_id=write_instance_id, host=bind_host, port=bind_port)
|
|
12631
|
+
write_descriptor_written = False
|
|
12632
|
+
try:
|
|
12633
|
+
_lib_write_io.remove_stale_descriptors(_cctally_core.APP_DIR)
|
|
12634
|
+
_lib_write_io.write_instance_descriptor(
|
|
12635
|
+
_cctally_core.APP_DIR, instance_id=write_instance_id,
|
|
12636
|
+
host=bind_host, port=bind_port)
|
|
12637
|
+
write_descriptor_written = True
|
|
12638
|
+
except OSError as exc:
|
|
12639
|
+
eprint(f"[dashboard] instance descriptor not written ({exc})")
|
|
12569
12640
|
|
|
12570
12641
|
# Surface any pending migration errors at server startup. Browser is a
|
|
12571
12642
|
# separate UI surface (out of scope for this fix); this prints once to
|
|
@@ -12609,6 +12680,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
12609
12680
|
finally:
|
|
12610
12681
|
# First, so a shutdown can never be followed by an automatic restart.
|
|
12611
12682
|
_stop_install_watch()
|
|
12683
|
+
if write_descriptor_written:
|
|
12684
|
+
try:
|
|
12685
|
+
_lib_write_io.remove_instance_descriptor(
|
|
12686
|
+
_cctally_core.APP_DIR, write_instance_id)
|
|
12687
|
+
except OSError:
|
|
12688
|
+
pass
|
|
12689
|
+
_lib_write_io.clear_local_instance()
|
|
12612
12690
|
if sync_thread is not None:
|
|
12613
12691
|
sync_thread.stop()
|
|
12614
12692
|
conversation_sync_stop.set()
|