cctally 1.113.0 → 1.114.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/README.md +4 -6
  3. package/bin/_cctally_cache.py +1683 -485
  4. package/bin/_cctally_core.py +223 -1
  5. package/bin/_cctally_dashboard.py +96 -18
  6. package/bin/_cctally_dashboard_perf.py +128 -9
  7. package/bin/_cctally_dashboard_sources.py +267 -204
  8. package/bin/_cctally_db.py +861 -18
  9. package/bin/_cctally_diagnosis_sources.py +22 -4
  10. package/bin/_cctally_doctor.py +136 -18
  11. package/bin/_cctally_five_hour.py +2 -2
  12. package/bin/_cctally_forecast.py +27 -6
  13. package/bin/_cctally_journal.py +67 -5
  14. package/bin/_cctally_milestone_history.py +82 -9
  15. package/bin/_cctally_percent_breakdown.py +3 -2
  16. package/bin/_cctally_quota.py +201 -33
  17. package/bin/_cctally_record.py +14 -0
  18. package/bin/_cctally_rederive.py +23 -9
  19. package/bin/_cctally_source_analytics.py +60 -15
  20. package/bin/_cctally_statusline.py +17 -0
  21. package/bin/_cctally_store.py +80 -5
  22. package/bin/_cctally_tui.py +4 -3
  23. package/bin/_cctally_weekrefs.py +128 -21
  24. package/bin/_lib_cache_report.py +35 -6
  25. package/bin/_lib_codex_conversation.py +32 -0
  26. package/bin/_lib_codex_conversation_query.py +164 -30
  27. package/bin/_lib_conversation_query.py +95 -56
  28. package/bin/_lib_conversation_retention.py +1605 -640
  29. package/bin/_lib_cost_provenance.py +53 -2
  30. package/bin/_lib_doctor.py +263 -69
  31. package/bin/_lib_fmt.py +11 -0
  32. package/bin/_lib_merge_gate.py +469 -51
  33. package/bin/_lib_perf.py +27 -1
  34. package/bin/_lib_pricing.py +193 -8
  35. package/bin/_lib_pricing_check.py +20 -4
  36. package/bin/_lib_reclaim_planner.py +792 -0
  37. package/bin/_lib_rederive.py +225 -77
  38. package/bin/_lib_snapshot_cache.py +21 -0
  39. package/bin/_lib_sqlite_close.py +81 -0
  40. package/bin/_lib_subscription_weeks.py +100 -17
  41. package/bin/_lib_tick_stats.py +217 -17
  42. package/bin/_lib_wal_checkpoint.py +1077 -0
  43. package/bin/_lib_write_budget.py +483 -0
  44. package/bin/_lib_write_io.py +703 -0
  45. package/bin/cctally +8 -0
  46. package/dashboard/static/assets/{ConversationsView-CEZ-BUyi.js → ConversationsView-Bc8LCOAZ.js} +1 -1
  47. package/dashboard/static/assets/{DoctorModal-CE1oH_Zy.js → DoctorModal-D--qVBwe.js} +1 -1
  48. package/dashboard/static/assets/ModalRoot-DLPgyAM_.js +1 -0
  49. package/dashboard/static/assets/{ProjectsDrillPanel-B3DZoCeD.js → ProjectsDrillPanel-z53a4v4M.js} +1 -1
  50. package/dashboard/static/assets/{SourceDetailModal-CHp9EgLP.js → SourceDetailModal-Bw0T8W-W.js} +1 -1
  51. package/dashboard/static/assets/{UpdateModal-5iKST6jo.js → UpdateModal-D5AKx4Yw.js} +1 -1
  52. package/dashboard/static/assets/index-BKITk1CT.css +1 -0
  53. package/dashboard/static/assets/{index-Cm0VTxJC.js → index-BXDOzSng.js} +3 -3
  54. package/dashboard/static/assets/{outlineNavigation-DkXGL8J6.js → outlineNavigation-JObY7cUY.js} +1 -1
  55. package/dashboard/static/dashboard.html +2 -2
  56. package/package.json +6 -1
  57. package/dashboard/static/assets/ModalRoot-D5Y8IFUG.js +0 -1
  58. package/dashboard/static/assets/index-Dt4tqf43.css +0 -1
@@ -473,7 +473,21 @@ _init_paths_from_env()
473
473
  # tick must carry that earlier evidence into the journal, even after this
474
474
  # binary learns a new model card. A pre-existing open projection that was
475
475
  # rebuilt without the marker remains unproven until a new tick recomputes it.
476
- STATS_INDEX_EPOCH = 1016
476
+ # 1016 -> 1017 (#901 §5.3, T3): read indexes only. Every stats digest relation
477
+ # (`_lib_dashboard_sources._CODEX_STATS_DIGEST_RELATIONS` and
478
+ # `_CLAUDE_STATS_DIGEST_RELATIONS`) sorted its whole relation through a temp
479
+ # b-tree on every dashboard build, and `_fetch_current_week_snapshots` grouped
480
+ # every boundary-aware and legacy-date snapshot the same way; each now streams
481
+ # through an index in its own order (`_STATS_READ_INDEX_DDL`). No table or
482
+ # column changes and no digest byte moves. The same epoch carries Amendment 1's
483
+ # history-read indexes (items 2-7 and 9, `c901-design-2`, with item 2's second
484
+ # single-root index from Amendment 1c, `c901-design-3`): every full dashboard
485
+ # build enumerated weekly blocks, milestone weeks, root keys, group pairs,
486
+ # reset events and subscription anchors through temp b-trees. The mechanical
487
+ # reason this is a bump and not a migration is unchanged: the registry is
488
+ # frozen at 13, and an epoch-current open returns before any schema work, so a
489
+ # schema helper alone would never run on an upgraded install.
490
+ STATS_INDEX_EPOCH = 1017
477
491
  LEGACY_STATS_HEAD = 13
478
492
 
479
493
  #: #496 S1 F1. A NEW branch, for a state that cannot occur before the
@@ -1861,6 +1875,207 @@ def _apply_quota_projection_schema(conn: sqlite3.Connection) -> None:
1861
1875
  "next_evaluation_by_root_json", "TEXT NOT NULL DEFAULT '{}'")
1862
1876
 
1863
1877
 
1878
+ #: #901 §5.3 (T3), epoch 1017: (table, columns the index reads, DDL). Each
1879
+ #: index's leading columns are the statement's equality filter (when it has
1880
+ #: one) followed by its ORDER BY / GROUP BY list, so the statement streams off
1881
+ #: the index with no sorter; the digest indexes are covering, because every
1882
+ #: digest relation selects exactly its ordering columns. SQLite stores each
1883
+ #: statement verbatim in `sqlite_schema`, which `_REBUILD_SCHEMA_FINGERPRINT`
1884
+ #: hashes: change the text and the fingerprint must move with it.
1885
+ _STATS_READ_INDEX_DDL: "tuple[tuple[str, tuple[str, ...], str], ...]" = (
1886
+ ("quota_projection_state", ("source_root_key", "physical_signature"),
1887
+ "CREATE INDEX IF NOT EXISTS idx_quota_projection_state_digest "
1888
+ "ON quota_projection_state(source_root_key, physical_signature)"),
1889
+ ("quota_window_blocks",
1890
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1891
+ "window_minutes", "limit_id", "limit_name", "resets_at_utc",
1892
+ "nominal_start_at_utc", "first_observed_at_utc", "last_observed_at_utc",
1893
+ "first_percent", "current_percent", "orphaned_at"),
1894
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_digest "
1895
+ "ON quota_window_blocks(source, source_root_key, logical_limit_key, "
1896
+ "observed_slot, window_minutes, limit_id, limit_name, resets_at_utc, "
1897
+ "nominal_start_at_utc, first_observed_at_utc, last_observed_at_utc, "
1898
+ "first_percent, current_percent, orphaned_at) WHERE source='codex'"),
1899
+ ("quota_percent_milestones",
1900
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1901
+ "window_minutes", "resets_at_utc", "percent_threshold",
1902
+ "captured_at_utc", "high_water_percent", "orphaned_at"),
1903
+ "CREATE INDEX IF NOT EXISTS idx_quota_milestones_digest "
1904
+ "ON quota_percent_milestones(source, source_root_key, logical_limit_key, "
1905
+ "observed_slot, window_minutes, resets_at_utc, percent_threshold, "
1906
+ "captured_at_utc, high_water_percent, orphaned_at) WHERE source='codex'"),
1907
+ ("quota_threshold_events",
1908
+ ("source", "source_root_key", "logical_limit_key", "observed_slot",
1909
+ "window_minutes", "resets_at_utc", "threshold", "qualifying_kind",
1910
+ "qualifying_percent", "projected_percent", "severity", "created_at_utc",
1911
+ "disposition", "alerted_at", "suppressed_at", "orphaned_at"),
1912
+ "CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_digest "
1913
+ "ON quota_threshold_events(source, source_root_key, logical_limit_key, "
1914
+ "observed_slot, window_minutes, resets_at_utc, threshold, "
1915
+ "qualifying_kind, qualifying_percent, projected_percent, severity, "
1916
+ "created_at_utc, disposition, alerted_at, suppressed_at, orphaned_at) "
1917
+ "WHERE source='codex'"),
1918
+ ("budget_milestones",
1919
+ ("vendor", "period_start_at", "period", "threshold", "budget_usd",
1920
+ "spent_usd", "consumption_pct", "crossed_at_utc", "alerted_at"),
1921
+ "CREATE INDEX IF NOT EXISTS idx_budget_milestones_codex_digest "
1922
+ "ON budget_milestones(vendor, period_start_at, period, threshold, "
1923
+ "budget_usd, spent_usd, consumption_pct, crossed_at_utc, alerted_at) "
1924
+ "WHERE vendor='codex'"),
1925
+ ("projected_milestones",
1926
+ ("metric", "week_start_at", "period", "threshold", "projected_value",
1927
+ "denominator", "crossed_at_utc", "alerted_at"),
1928
+ "CREATE INDEX IF NOT EXISTS idx_projected_milestones_codex_digest "
1929
+ "ON projected_milestones(metric, week_start_at, period, threshold, "
1930
+ "projected_value, denominator, crossed_at_utc, alerted_at) "
1931
+ "WHERE metric='codex_budget_usd'"),
1932
+ ("percent_milestones",
1933
+ ("week_start_date", "percent_threshold", "reset_event_id", "account_key",
1934
+ "captured_at_utc", "cumulative_cost_usd", "alerted_at"),
1935
+ "CREATE INDEX IF NOT EXISTS idx_percent_milestones_alert_digest "
1936
+ "ON percent_milestones(week_start_date, percent_threshold, "
1937
+ "reset_event_id, account_key, captured_at_utc, cumulative_cost_usd, "
1938
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
1939
+ ("five_hour_milestones",
1940
+ ("five_hour_window_key", "percent_threshold", "reset_event_id",
1941
+ "account_key", "captured_at_utc", "block_cost_usd", "alerted_at"),
1942
+ "CREATE INDEX IF NOT EXISTS idx_five_hour_milestones_alert_digest "
1943
+ "ON five_hour_milestones(five_hour_window_key, percent_threshold, "
1944
+ "reset_event_id, account_key, captured_at_utc, block_cost_usd, "
1945
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
1946
+ ("budget_milestones",
1947
+ ("vendor", "period_start_at", "period", "threshold", "account_key",
1948
+ "budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
1949
+ "alerted_at"),
1950
+ "CREATE INDEX IF NOT EXISTS idx_budget_milestones_alert_digest "
1951
+ "ON budget_milestones(vendor, period_start_at, period, threshold, "
1952
+ "account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
1953
+ "alerted_at) WHERE vendor <> 'codex' AND alerted_at IS NOT NULL"),
1954
+ ("projected_milestones",
1955
+ ("week_start_at", "period", "metric", "threshold", "account_key",
1956
+ "projected_value", "denominator", "crossed_at_utc", "alerted_at"),
1957
+ "CREATE INDEX IF NOT EXISTS idx_projected_milestones_alert_digest "
1958
+ "ON projected_milestones(week_start_at, period, metric, threshold, "
1959
+ "account_key, projected_value, denominator, crossed_at_utc, alerted_at) "
1960
+ "WHERE metric <> 'codex_budget_usd' AND alerted_at IS NOT NULL"),
1961
+ ("project_budget_milestones",
1962
+ ("week_start_at", "project_key", "threshold", "account_key",
1963
+ "budget_usd", "spent_usd", "consumption_pct", "crossed_at_utc",
1964
+ "alerted_at"),
1965
+ "CREATE INDEX IF NOT EXISTS idx_project_budget_milestones_alert_digest "
1966
+ "ON project_budget_milestones(week_start_at, project_key, threshold, "
1967
+ "account_key, budget_usd, spent_usd, consumption_pct, crossed_at_utc, "
1968
+ "alerted_at) WHERE alerted_at IS NOT NULL"),
1969
+ ("weekly_usage_snapshots",
1970
+ ("week_start_at", "week_end_at", "week_start_date", "account_key",
1971
+ "weekly_observation_held", "captured_at_utc"),
1972
+ "CREATE INDEX IF NOT EXISTS idx_usage_week_boundary_group "
1973
+ "ON weekly_usage_snapshots(week_start_at, week_end_at, week_start_date, "
1974
+ "account_key, weekly_observation_held, captured_at_utc) "
1975
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
1976
+ ("weekly_usage_snapshots",
1977
+ ("week_start_date", "week_end_date", "account_key",
1978
+ "weekly_observation_held", "captured_at_utc"),
1979
+ "CREATE INDEX IF NOT EXISTS idx_usage_week_date_group "
1980
+ "ON weekly_usage_snapshots(week_start_date, week_end_date, account_key, "
1981
+ "weekly_observation_held, captured_at_utc)"),
1982
+ # Amendment 1 (binding Codex answer `c901-design-2`): the recurring history
1983
+ # reads the #901 plan guard escalated. Item 2: `_load_codex_cycles` orders
1984
+ # every weekly block of the active roots by reset; the partial predicate
1985
+ # deliberately keeps orphaned rows so `include_orphaned=True` streams too.
1986
+ # Amendment 1b widened it past the reset second: `_canonicalize_codex_
1987
+ # cluster` keeps the first maximal member, so the order of rows tied on one
1988
+ # second is observable, and the trailing columns make it explicit. The
1989
+ # table's UNIQUE constraint makes them unique within the population.
1990
+ # Amendment 1c (`c901-design-3`): the legacy plan fed its sorter in two
1991
+ # different orders. A read of two or more roots scanned
1992
+ # `idx_quota_blocks_active` (ties by root, account, orphaned_at, limit,
1993
+ # slot, reset spelling); a read of exactly one root seeked the UNIQUE
1994
+ # autoindex (the same order without orphaned_at). The first index below
1995
+ # serves the multi-root read, the second the single-root read, each in its
1996
+ # legacy tie order.
1997
+ ("quota_window_blocks",
1998
+ ("source", "window_minutes", "resets_at_utc", "source_root_key",
1999
+ "account_key", "orphaned_at", "logical_limit_key", "observed_slot"),
2000
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_reset_order "
2001
+ "ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
2002
+ "source_root_key ASC, account_key ASC, orphaned_at ASC, "
2003
+ "logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
2004
+ "WHERE source='codex' AND window_minutes=10080"),
2005
+ ("quota_window_blocks",
2006
+ ("source", "window_minutes", "resets_at_utc", "source_root_key",
2007
+ "account_key", "logical_limit_key", "observed_slot"),
2008
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_weekly_single_root_order "
2009
+ "ON quota_window_blocks(unixepoch(resets_at_utc) DESC, "
2010
+ "source_root_key ASC, account_key ASC, "
2011
+ "logical_limit_key ASC, observed_slot ASC, resets_at_utc ASC) "
2012
+ "WHERE source='codex' AND window_minutes=10080"),
2013
+ # Item 3: `_navigable_claude_refs` de-duplicates every milestone week; a
2014
+ # full index, because the alert digest index cannot see unalerted rows.
2015
+ ("percent_milestones", ("week_start_date",),
2016
+ "CREATE INDEX IF NOT EXISTS idx_percent_milestones_week_date "
2017
+ "ON percent_milestones(week_start_date ASC)"),
2018
+ # Item 4: `_historic_root_keys` reads the complete relations.
2019
+ ("quota_percent_milestones", ("source_root_key",),
2020
+ "CREATE INDEX IF NOT EXISTS idx_quota_milestones_root "
2021
+ "ON quota_percent_milestones(source_root_key ASC)"),
2022
+ ("quota_threshold_events", ("source_root_key",),
2023
+ "CREATE INDEX IF NOT EXISTS idx_quota_threshold_events_root "
2024
+ "ON quota_threshold_events(source_root_key ASC)"),
2025
+ # Item 5 (and item 4's quota_window_blocks leg): one root's distinct
2026
+ # (group key, group digest) pairs stream after three equality columns.
2027
+ ("quota_window_blocks",
2028
+ ("source_root_key", "source", "orphaned_at", "physical_group_key",
2029
+ "physical_group_digest"),
2030
+ "CREATE INDEX IF NOT EXISTS idx_quota_blocks_root_group_pairs "
2031
+ "ON quota_window_blocks(source_root_key ASC, source ASC, orphaned_at ASC, "
2032
+ "physical_group_key ASC, physical_group_digest ASC)"),
2033
+ # Items 6 and 7: the WeekRef reset-event applier and
2034
+ # `in_place_cut_instants` read every event in the chokepoint's order.
2035
+ ("week_reset_events", ("effective_reset_at_utc", "id"),
2036
+ "CREATE INDEX IF NOT EXISTS idx_week_reset_events_effective_order "
2037
+ "ON week_reset_events(unixepoch(effective_reset_at_utc) DESC, id DESC)"),
2038
+ # Item 9: `_compute_subscription_weeks` scans representative rows in
2039
+ # (minimum start, date) order and picks and aggregates each date's
2040
+ # qualifying population. The order index's predicate omits the answer's
2041
+ # `week_start_date IS NOT NULL`: the column is NOT NULL, so SQLite drops
2042
+ # that term from the outer scan's WHERE and then cannot prove the partial
2043
+ # index usable ("no query solution" under INDEXED BY, measured). The
2044
+ # indexed population is identical. The pick index keeps the term, because
2045
+ # every lookup through it carries a date equality that implies it.
2046
+ ("weekly_usage_snapshots",
2047
+ ("week_start_at", "week_end_at", "week_start_date"),
2048
+ "CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_order "
2049
+ "ON weekly_usage_snapshots(week_start_at ASC, week_start_date ASC) "
2050
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL"),
2051
+ ("weekly_usage_snapshots",
2052
+ ("week_start_date", "week_start_at", "week_end_at", "id"),
2053
+ "CREATE INDEX IF NOT EXISTS idx_usage_subscription_anchor_pick "
2054
+ "ON weekly_usage_snapshots(week_start_date ASC, week_start_at ASC, id ASC) "
2055
+ "WHERE week_start_at IS NOT NULL AND week_end_at IS NOT NULL "
2056
+ "AND week_start_date IS NOT NULL"),
2057
+ )
2058
+
2059
+
2060
+ def _apply_stats_read_indexes(conn: sqlite3.Connection) -> None:
2061
+ """Create each #901 read index whose table and columns exist (epoch 1017).
2062
+
2063
+ ``open_db`` calls this twice: beside the quota projection schema, and
2064
+ again after the stats migration dispatcher, because a legacy store being
2065
+ cut over gains one indexed column (``reset_event_id``, migration 005) only
2066
+ from the dispatcher. Every statement is IF NOT EXISTS, so the second call
2067
+ creates exactly what the first had to skip, and an index is never attempted
2068
+ over a column that is not there yet.
2069
+ """
2070
+ for table, required, statement in _STATS_READ_INDEX_DDL:
2071
+ columns = {
2072
+ str(row[0]) for row in conn.execute(
2073
+ "SELECT name FROM pragma_table_info(?)", (table,))
2074
+ }
2075
+ if set(required) <= columns:
2076
+ conn.execute(statement)
2077
+
2078
+
1864
2079
  def open_db(*, _target_path=None) -> sqlite3.Connection:
1865
2080
  # ``_target_path`` (internal, keyword-only) builds/opens the stats index at
1866
2081
  # an ALTERNATE path instead of the module-global ``DB_PATH`` — the seam
@@ -2129,6 +2344,10 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2129
2344
  conn, _STATS_MIGRATIONS, "stats.db"
2130
2345
  )
2131
2346
  return conn
2347
+ # PR-3: everything below is pending schema, migration or cutover work;
2348
+ # it sorts under FILE, and the writer temp store is selected again
2349
+ # before the connection is returned.
2350
+ _cctally_store.apply_schema_work_temp_store(conn)
2132
2351
  _fixups_current = _cctally_store.stats_open_fixups_current(conn)
2133
2352
  conn.execute(
2134
2353
  """
@@ -2854,6 +3073,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2854
3073
  # without running its handler, so this open-time call is the sole creator).
2855
3074
  if not _fixups_current:
2856
3075
  _apply_quota_projection_schema(conn)
3076
+ _apply_stats_read_indexes(conn)
2857
3077
 
2858
3078
  # Migration framework dispatcher. Replaces the prior inline gate stack
2859
3079
  # (has_blocks + _migration_done) with the framework's _run_pending_-
@@ -2871,6 +3091,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
2871
3091
  _run_pending_migrations(
2872
3092
  conn, registry=_STATS_MIGRATIONS, db_label="stats.db",
2873
3093
  )
3094
+ _apply_stats_read_indexes(conn)
2874
3095
 
2875
3096
  # One-time historical backfill of five_hour_blocks (rollup only;
2876
3097
  # milestones are forward-only per spec §4.3 / [Write-once milestones]).
@@ -3186,6 +3407,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
3186
3407
  except Exception:
3187
3408
  pass
3188
3409
  raise
3410
+ _cctally_store.apply_writer_temp_store(conn)
3189
3411
  return conn
3190
3412
 
3191
3413
 
@@ -764,6 +764,7 @@ from _lib_snapshot_cache import (
764
764
  _reset_sig as _snapshot_reset_sig,
765
765
  )
766
766
  import _lib_tick_stats
767
+ import _lib_write_io
767
768
 
768
769
 
769
770
  # === F22a: count the silent Group A cache-open failures (#583 S1 §1.6) =====
@@ -2855,11 +2856,13 @@ def _conversation_sync_loop(
2855
2856
  while not stop.is_set():
2856
2857
  t0 = monotonic()
2857
2858
  cpu0 = thread_time_ns()
2859
+ written = _lib_write_io.begin_interval("conversation")
2858
2860
  try:
2859
2861
  status = run_iteration() or "ok"
2860
2862
  except Exception: # noqa: BLE001 — the worker must outlive one bad pass
2861
2863
  _log_sync_iteration_failure()
2862
2864
  status = "error"
2865
+ observation = _lib_write_io.end_interval(written)
2863
2866
  work = max(0.0, monotonic() - t0)
2864
2867
  cpu_ns = max(0, thread_time_ns() - cpu0)
2865
2868
  seq += 1
@@ -2882,6 +2885,12 @@ def _conversation_sync_loop(
2882
2885
  "codex", "not_observed"),
2883
2886
  claude_files=getattr(status, "files", {}).get("claude", 0),
2884
2887
  codex_files=getattr(status, "files", {}).get("codex", 0),
2888
+ # #901 §5.5: the pass's process write delta. It includes the
2889
+ # main loop's concurrent work, which `write_overlap` flags; it
2890
+ # is never summed with a tick's.
2891
+ process_write_bytes=observation.process_write_bytes,
2892
+ write_status=observation.status,
2893
+ write_overlap=observation.overlap,
2885
2894
  )
2886
2895
  deadline = _conversation_next_deadline(t0, interval, work)
2887
2896
  remaining = deadline - monotonic()
@@ -6894,6 +6903,38 @@ def _assemble_projects_via_cache(
6894
6903
  return buckets, total_cost_by_week, key_by_bucket
6895
6904
 
6896
6905
 
6906
+ def _projects_week_usage_rows(
6907
+ conn: "sqlite3.Connection", since_date: str, end_date: str,
6908
+ ) -> list:
6909
+ """The anchored, non-NULL weekly readings of the projects window, oldest
6910
+ capture first: ``(week_start_date, week_start_at, weekly_percent,
6911
+ captured_at_utc, id)`` for every week date in ``[since_date, end_date)``.
6912
+
6913
+ The window holds ``weeks_back`` weeks of status-line readings, which is
6914
+ not a bound on a sorter, so the rows are ordered here rather than by an
6915
+ ``ORDER BY`` that built a temp b-tree over all of them on every build
6916
+ (#901 Q11). ``(captured_at_utc, id)`` is a total order — ``id`` is the
6917
+ primary key and ``captured_at_utc`` is NOT NULL TEXT, which Python
6918
+ compares in the same code-point order as SQLite's BINARY collation — so
6919
+ the result is exactly the former ``ORDER BY captured_at_utc ASC, id ASC``.
6920
+ """
6921
+ rows = conn.execute(
6922
+ " SELECT week_start_date, week_start_at, weekly_percent,"
6923
+ " captured_at_utc, id"
6924
+ " FROM weekly_usage_snapshots"
6925
+ # Held rows excluded: these rows feed `latest_usage_by_segment`, which
6926
+ # is a weekly read, and the pre-fetch must select the same population
6927
+ # the reducer would select for itself (#769 S11, #824).
6928
+ " WHERE week_start_date >= ? AND week_start_date < ?"
6929
+ " AND date(week_start_date) IS NOT NULL"
6930
+ " AND weekly_percent IS NOT NULL"
6931
+ + _cctally_core.weekly_held_exclusion(conn),
6932
+ (since_date, end_date),
6933
+ ).fetchall()
6934
+ rows.sort(key=lambda row: (row[3], row[4]))
6935
+ return rows
6936
+
6937
+
6897
6938
  def _build_projects_envelope(
6898
6939
  conn: "sqlite3.Connection",
6899
6940
  *,
@@ -7165,24 +7206,8 @@ def _build_projects_envelope(
7165
7206
  # valid rendered-week row (#620 S1 A3).
7166
7207
  weekly_pct_by_week: dict[dt.datetime, float] = {}
7167
7208
  try:
7168
- cur = conn.execute(
7169
- " SELECT week_start_date, week_start_at, weekly_percent,"
7170
- " captured_at_utc, id"
7171
- " FROM weekly_usage_snapshots"
7172
- # Held rows excluded: these rows feed `latest_usage_by_segment`,
7173
- # which is a weekly read, and the pre-fetch must select the same
7174
- # population the reducer would select for itself (#769 S11, #824).
7175
- " WHERE week_start_date >= ? AND week_start_date < ?"
7176
- " AND date(week_start_date) IS NOT NULL"
7177
- " AND weekly_percent IS NOT NULL"
7178
- + _cctally_core.weekly_held_exclusion(conn) +
7179
- " ORDER BY captured_at_utc ASC, id ASC",
7180
- (
7181
- since_dt.date().isoformat(),
7182
- cw_end.date().isoformat(),
7183
- ),
7184
- )
7185
- rows = cur.fetchall()
7209
+ rows = _projects_week_usage_rows(
7210
+ conn, since_dt.date().isoformat(), cw_end.date().isoformat())
7186
7211
  except sqlite3.OperationalError:
7187
7212
  # No weekly_usage_snapshots table — leaves attributed_pct = None
7188
7213
  # throughout (acceptable per spec §2.7).
@@ -8086,6 +8111,15 @@ class _DiagnosisSelectorError(Exception):
8086
8111
 
8087
8112
 
8088
8113
  # ── /api/debug/backend on-demand cache-state helpers (issue #276, Session A) ──
8114
+
8115
+ def _debug_write_io() -> dict:
8116
+ """`writeIo` for the debug surfaces (#901 §5.5). A telemetry fault never
8117
+ takes the diagnostic down, and is never reported as a zero."""
8118
+ try:
8119
+ return _lib_write_io.write_io_payload()
8120
+ except Exception: # noqa: BLE001 — diagnostics fail closed
8121
+ return {"status": "unavailable", "reason": "counter_error"}
8122
+
8089
8123
  # All read-only, cheap, and privacy-safe: they leak ONLY row counts, signature
8090
8124
  # legs (ints/tuples), pending-flag names, and the tool version — never prompt /
8091
8125
  # prose / paths. Computed on demand so cache_state is available even with
@@ -8494,6 +8528,7 @@ _GET_ROUTES = (
8494
8528
  ("exact", "/api/diagnosis", "_handle_get_diagnosis",
8495
8529
  ("scope", "endpoint.diagnosis"), False),
8496
8530
  ("exact", "/api/debug/backend", "_handle_get_debug_backend", None, False),
8531
+ ("exact", "/api/debug/backend/write-io", "_handle_get_debug_backend_write_io", None, False),
8497
8532
  ("exact", "/api/conversations/facets", "_handle_get_conversations_facets",
8498
8533
  ("scope", "endpoint.conversations_facets"), False),
8499
8534
  ("exact", "/api/conversations", "_handle_get_conversations",
@@ -9224,11 +9259,26 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
9224
9259
  # stats failure reported as a cache one sends the user to
9225
9260
  # `cctally cache-sync --rebuild`.
9226
9261
  "stats_faults": stats_faults,
9262
+ # #901 §5.5: kernel-accounted process writes, the trailing
9263
+ # five-minute statistic and its budget verdict. Additive camelCase.
9264
+ "writeIo": _debug_write_io(),
9227
9265
  }
9228
9266
  if body["phases"] is None:
9229
9267
  body["note"] = "tracing_disabled"
9230
9268
  self._respond_json(200, body)
9231
9269
 
9270
+ def _handle_get_debug_backend_write_io(self) -> None:
9271
+ """GET ``/api/debug/backend/write-io`` — the ``writeIo`` object alone
9272
+ (#901 §5.6): what ``cctally doctor`` reads from a running dashboard
9273
+ inside its bounded timeout, without ``/api/debug/backend``'s
9274
+ on-demand cache-table counts. Same gates, same order: the bearer
9275
+ (``_require_api_auth``, before dispatch), then the loopback TCP peer
9276
+ and the IP-literal ``Host``. No bypass exists."""
9277
+ if not self._require_debug_backend_allowed():
9278
+ return
9279
+ self._respond_json(200, {"schemaVersion": 1,
9280
+ "writeIo": _debug_write_io()})
9281
+
9232
9282
  def _handle_post_debug_backend_trace(self) -> None:
9233
9283
  """POST ``/api/debug/backend/trace`` — arm the deep phase trace (§3.2).
9234
9284
 
@@ -12485,6 +12535,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
12485
12535
  **_make_sync_loop_collaborators(ref=ref, hub=hub),
12486
12536
  )
12487
12537
 
12538
+ # #901 §5.6: this process's write-telemetry identity, registered before
12539
+ # the first build so the in-process doctor reads its own snapshot from the
12540
+ # first tick; the bound port replaces the requested one once known.
12541
+ write_instance_id = _lib_write_io.new_instance_id()
12542
+ _lib_write_io.register_local_instance(
12543
+ instance_id=write_instance_id, host=args.host,
12544
+ port=_resolve_dashboard_port(args.port))
12488
12545
  sync_thread = (
12489
12546
  None if args.no_sync
12490
12547
  else _DashboardSyncThread(
@@ -12566,6 +12623,20 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
12566
12623
  http_thread = threading.Thread(target=srv.serve_forever, daemon=True,
12567
12624
  name="dashboard-http")
12568
12625
  http_thread.start()
12626
+ # The owner-readable descriptor `cctally doctor` discovers this instance
12627
+ # through: written once now, removed at clean shutdown. A crash leaves it
12628
+ # for the next startup to remove; doctor ignores a dead or reused pid's.
12629
+ _lib_write_io.register_local_instance(
12630
+ instance_id=write_instance_id, host=bind_host, port=bind_port)
12631
+ write_descriptor_written = False
12632
+ try:
12633
+ _lib_write_io.remove_stale_descriptors(_cctally_core.APP_DIR)
12634
+ _lib_write_io.write_instance_descriptor(
12635
+ _cctally_core.APP_DIR, instance_id=write_instance_id,
12636
+ host=bind_host, port=bind_port)
12637
+ write_descriptor_written = True
12638
+ except OSError as exc:
12639
+ eprint(f"[dashboard] instance descriptor not written ({exc})")
12569
12640
 
12570
12641
  # Surface any pending migration errors at server startup. Browser is a
12571
12642
  # separate UI surface (out of scope for this fix); this prints once to
@@ -12609,6 +12680,13 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
12609
12680
  finally:
12610
12681
  # First, so a shutdown can never be followed by an automatic restart.
12611
12682
  _stop_install_watch()
12683
+ if write_descriptor_written:
12684
+ try:
12685
+ _lib_write_io.remove_instance_descriptor(
12686
+ _cctally_core.APP_DIR, write_instance_id)
12687
+ except OSError:
12688
+ pass
12689
+ _lib_write_io.clear_local_instance()
12612
12690
  if sync_thread is not None:
12613
12691
  sync_thread.stop()
12614
12692
  conversation_sync_stop.set()