cctally 1.92.2 → 1.93.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/bin/_cctally_cache.py +354 -0
- package/bin/_cctally_core.py +180 -3
- package/bin/_cctally_dashboard.py +71 -1
- package/bin/_cctally_dashboard_envelope.py +28 -2
- package/bin/_cctally_dashboard_share.py +75 -19
- package/bin/_cctally_dashboard_sources.py +12 -0
- package/bin/_cctally_db.py +89 -1
- package/bin/_cctally_doctor.py +31 -0
- package/bin/_cctally_forecast.py +4 -2
- package/bin/_cctally_journal.py +3482 -258
- package/bin/_cctally_journal_repair.py +123 -32
- package/bin/_cctally_milestone_history.py +4 -1
- package/bin/_cctally_project.py +8 -6
- package/bin/_cctally_quota.py +420 -20
- package/bin/_cctally_rederive.py +57 -23
- package/bin/_cctally_reporting.py +8 -6
- package/bin/_cctally_share.py +74 -37
- package/bin/_cctally_source_analytics.py +6 -8
- package/bin/_cctally_store.py +13 -2
- package/bin/_cctally_tui.py +53 -0
- package/bin/_lib_cache_coverage.py +547 -0
- package/bin/_lib_doctor.py +54 -2
- package/bin/_lib_journal.py +235 -95
- package/bin/_lib_journal_router.py +21 -0
- package/bin/_lib_segment_summary.py +374 -0
- package/bin/_lib_selector_state.py +959 -0
- package/bin/_lib_share.py +1073 -165
- package/bin/_lib_share_templates.py +35 -11
- package/bin/_lib_stats_wal.py +327 -0
- package/bin/_lib_view_models.py +2 -1
- package/dashboard/static/assets/index-DwWJOYxd.css +1 -0
- package/dashboard/static/assets/{index-Dat-mza6.js → index-HlIK7k8Q.js} +47 -47
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +5 -1
- package/dashboard/static/assets/index-DnWdv8um.css +0 -1
package/bin/_cctally_core.py
CHANGED
|
@@ -375,7 +375,16 @@ STATS_WAL_SIZE_LIMIT_BYTES = 16 * 1024 * 1024 # 16777216
|
|
|
375
375
|
# rolled back. A stats schema change is an epoch bump and never a migration;
|
|
376
376
|
# the 13-migration registry stays frozen. Each install pays one rebuild on
|
|
377
377
|
# upgrade, deferred to the background worker by #453.
|
|
378
|
-
|
|
378
|
+
# 1008 -> 1009 (#496 S5b): durable replay selection. Adds the three
|
|
379
|
+
# `journal_selector_*` tables reproducing `resolve_effective_events`' six
|
|
380
|
+
# accumulators, `stats_quota_projection_state` (reserved by Stage 1, set by
|
|
381
|
+
# Stage 3), plus `journal_effective_events.winning_sequence` /
|
|
382
|
+
# `.conflict_hashes_json` and `journal_protocol_violations.available_after`.
|
|
383
|
+
# Same mechanical reason as every bump since 1005: an epoch-current open returns
|
|
384
|
+
# before any schema work, so an `add_column_if_missing` would never run on an
|
|
385
|
+
# upgraded install and the column would simply never appear. The registry stays
|
|
386
|
+
# frozen at 13 and an epoch mismatch resolves by rebuild.
|
|
387
|
+
STATS_INDEX_EPOCH = 1009
|
|
379
388
|
LEGACY_STATS_HEAD = 13
|
|
380
389
|
|
|
381
390
|
#: #496 S1 F1. A NEW branch, for a state that cannot occur before the
|
|
@@ -777,6 +786,43 @@ def holds_stats_maintenance() -> bool:
|
|
|
777
786
|
return _STATS_MAINTENANCE_HELD.get() > 0
|
|
778
787
|
|
|
779
788
|
|
|
789
|
+
# === #496 S5b §4.7 open-time quota-projection reconciliation (opt-in) ======
|
|
790
|
+
#
|
|
791
|
+
# A rebuild that published a valid index over a cache with an uncovered
|
|
792
|
+
# remainder durably marks its quota projection incomplete, and some later open
|
|
793
|
+
# has to resume that recovery or the gate never lifts. `open_db` is where that
|
|
794
|
+
# resumption lives, but it is NOT something every open may pay: `open_db` runs
|
|
795
|
+
# on `cctally statusline` and on every hook tick, and the resumption reads the
|
|
796
|
+
# journal from zero to the current high water — the 1.64 GB working set the S4
|
|
797
|
+
# measurements describe — before it can apply anything.
|
|
798
|
+
#
|
|
799
|
+
# So the trigger is armed per process rather than unconditional. A maintenance
|
|
800
|
+
# command that is already doing journal-scale work arms it; the interactive
|
|
801
|
+
# render paths never do, and pay only the one indexed flag SELECT that was
|
|
802
|
+
# already there. A process global rather than a ContextVar deliberately: this
|
|
803
|
+
# is a property of the COMMAND that is running, not of one thread inside it,
|
|
804
|
+
# and the dashboard arms it once for the whole server.
|
|
805
|
+
QUOTA_PROJECTION_RECONCILE_ENABLED = False
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def enable_quota_projection_reconciliation() -> None:
|
|
809
|
+
"""Arm the open-time quota-projection reconciliation for this process.
|
|
810
|
+
|
|
811
|
+
A PROCESS global, deliberately, and not a ContextVar or a thread-local: the
|
|
812
|
+
two armers are `cmd_cache_sync` and `cmd_dashboard`, and the dashboard has
|
|
813
|
+
no dedicated maintenance thread to hang a thread-local on. So in the
|
|
814
|
+
dashboard whichever thread reaches `open_db` first runs the attempt, and one
|
|
815
|
+
attempt can stall that thread while it holds the maintenance flock. That is
|
|
816
|
+
accepted rather than overlooked: the attempt takes every lock
|
|
817
|
+
non-blocking and returns immediately when any is busy, the throttle bounds
|
|
818
|
+
the repeat to one per interval, and the alternative — routing this through a
|
|
819
|
+
worker the dashboard does not currently have — is a larger change than the
|
|
820
|
+
contention it would avoid.
|
|
821
|
+
"""
|
|
822
|
+
global QUOTA_PROJECTION_RECONCILE_ENABLED
|
|
823
|
+
QUOTA_PROJECTION_RECONCILE_ENABLED = True
|
|
824
|
+
|
|
825
|
+
|
|
780
826
|
# === stats.db sanctioned-write scope (#386) =========================
|
|
781
827
|
#
|
|
782
828
|
# The state behind `_cctally_store.stats_write_scope` / `in_stats_write_scope`
|
|
@@ -1537,6 +1583,35 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1537
1583
|
_reconcile_durable_applied_migration_errors = (
|
|
1538
1584
|
c._reconcile_durable_applied_migration_errors
|
|
1539
1585
|
)
|
|
1586
|
+
|
|
1587
|
+
def _reconcile_incomplete_quota_projection(_conn):
|
|
1588
|
+
"""#496 S5b §4.7's open-time gate, reached only on the steady-state path.
|
|
1589
|
+
|
|
1590
|
+
A rebuild that published a valid index over a cache with an uncovered
|
|
1591
|
+
remainder durably marks its quota projection incomplete, and every
|
|
1592
|
+
projection read is then gated. Some open has to be able to resume that
|
|
1593
|
+
recovery, or the gate never lifts. Everything past the flag probe lives
|
|
1594
|
+
in `_cctally_journal`, which owns the recovery leg and the lock order;
|
|
1595
|
+
the reach is call-time for the same reason the opener policy's is.
|
|
1596
|
+
|
|
1597
|
+
ARMED PROCESSES ONLY. The resumption reads the whole journal, and this
|
|
1598
|
+
function is reached by `cctally statusline` and by every hook tick. An
|
|
1599
|
+
unarmed process returns here, before the `_cctally_journal` import — so
|
|
1600
|
+
an interactive render pays neither the import nor the read. See
|
|
1601
|
+
`enable_quota_projection_reconciliation`.
|
|
1602
|
+
|
|
1603
|
+
Any failure is swallowed. This runs on the hot open path, and a
|
|
1604
|
+
reconciliation that cannot proceed must leave the flag set — the
|
|
1605
|
+
fail-closed direction — rather than fail an unrelated command.
|
|
1606
|
+
"""
|
|
1607
|
+
if not QUOTA_PROJECTION_RECONCILE_ENABLED:
|
|
1608
|
+
return
|
|
1609
|
+
try:
|
|
1610
|
+
import importlib as _il
|
|
1611
|
+
_il.import_module(
|
|
1612
|
+
"_cctally_journal").reconcile_incomplete_quota_projection(_conn)
|
|
1613
|
+
except Exception:
|
|
1614
|
+
return
|
|
1540
1615
|
# Unified opener policy (spec §6.1). Call-time import so the shared PRAGMA
|
|
1541
1616
|
# policy applies without a module-load cycle (_cctally_store imports this
|
|
1542
1617
|
# module). Routed through importlib.import_module rather than a bare
|
|
@@ -1663,6 +1738,7 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
1663
1738
|
_reconcile_durable_applied_migration_errors(
|
|
1664
1739
|
conn, _STATS_MIGRATIONS, "stats.db"
|
|
1665
1740
|
)
|
|
1741
|
+
_reconcile_incomplete_quota_projection(conn)
|
|
1666
1742
|
return conn
|
|
1667
1743
|
if _target_path is None:
|
|
1668
1744
|
if _uv > LEGACY_STATS_HEAD:
|
|
@@ -2488,6 +2564,15 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2488
2564
|
# the append-only journal; rebuild repopulates this table from the shared
|
|
2489
2565
|
# pure selector. The table lets live replay detect completed corrections
|
|
2490
2566
|
# without inventing family-specific inverse operations.
|
|
2567
|
+
# `winning_sequence` and `conflict_hashes_json` (#496 S5b §3.2) complete
|
|
2568
|
+
# this table into the selector's whole per-event accumulator, which is
|
|
2569
|
+
# why S5b adds NO per-candidate table: same-revision containment needs
|
|
2570
|
+
# only the winning revision, the lowest-sequence winner and the set of
|
|
2571
|
+
# distinct content hashes observed at that revision, and this table is
|
|
2572
|
+
# already keyed one row per event id. The columns are declared HERE
|
|
2573
|
+
# rather than added by `add_column_if_missing`, because this is an epoch
|
|
2574
|
+
# bump: an epoch-current open returns before any schema work, so a
|
|
2575
|
+
# conditional column addition would never run on an upgraded install.
|
|
2491
2576
|
conn.execute(
|
|
2492
2577
|
"CREATE TABLE IF NOT EXISTS journal_effective_events ("
|
|
2493
2578
|
"event_id TEXT PRIMARY KEY, "
|
|
@@ -2495,17 +2580,109 @@ def open_db(*, _target_path=None) -> sqlite3.Connection:
|
|
|
2495
2580
|
"status TEXT NOT NULL CHECK (status IN ('active','tombstone')), "
|
|
2496
2581
|
"content_hash TEXT NOT NULL, "
|
|
2497
2582
|
"batch_id TEXT, "
|
|
2498
|
-
"event_json TEXT
|
|
2583
|
+
"event_json TEXT, "
|
|
2584
|
+
"winning_sequence INTEGER, "
|
|
2585
|
+
"conflict_hashes_json TEXT)"
|
|
2499
2586
|
)
|
|
2500
2587
|
# Disposable selector diagnostics (#402 Task A). The append-only journal
|
|
2501
2588
|
# remains authoritative; rebuild/live preflight replace this bounded
|
|
2502
2589
|
# summary after a complete correction-prefix selection.
|
|
2590
|
+
#
|
|
2591
|
+
# `available_after` (#496 S5b §3.2) is the selector's per-fingerprint
|
|
2592
|
+
# minimum sequence: a `journal_protocol_resolution` op that precedes it
|
|
2593
|
+
# is fatal, so an incremental pass that cannot re-derive it cannot
|
|
2594
|
+
# decide whether a resolution is legitimate.
|
|
2503
2595
|
conn.execute(
|
|
2504
2596
|
"CREATE TABLE IF NOT EXISTS journal_protocol_violations ("
|
|
2505
2597
|
"fingerprint TEXT PRIMARY KEY, "
|
|
2506
2598
|
"batch_id TEXT NOT NULL, "
|
|
2507
2599
|
"kind TEXT NOT NULL, "
|
|
2508
|
-
"violation_json TEXT NOT NULL
|
|
2600
|
+
"violation_json TEXT NOT NULL, "
|
|
2601
|
+
"available_after INTEGER)"
|
|
2602
|
+
)
|
|
2603
|
+
# The primary key is the fingerprint, but every scoped selector read
|
|
2604
|
+
# filters `WHERE batch_id IN (...)` — on the status-line path, once per
|
|
2605
|
+
# merge tick. Without this index that is a full table scan, and the
|
|
2606
|
+
# rows-read guard in `tests/test_live_selection_496_s5b.py` cannot see
|
|
2607
|
+
# it: that proxy counts rows RETURNED, not rows scanned.
|
|
2608
|
+
conn.execute(
|
|
2609
|
+
"CREATE INDEX IF NOT EXISTS idx_journal_protocol_violations_batch "
|
|
2610
|
+
"ON journal_protocol_violations(batch_id)"
|
|
2611
|
+
)
|
|
2612
|
+
# ── Durable selector state (#496 S5b §3.2) ──────────────────────────
|
|
2613
|
+
# `resolve_effective_events` accumulates six things over the record
|
|
2614
|
+
# stream and returns only a summary, so every live tick that meets a
|
|
2615
|
+
# correction record re-derives them from the whole journal prefix. These
|
|
2616
|
+
# three tables reproduce those six accumulators and nothing more, which
|
|
2617
|
+
# is what lets a validated generation seed incrementally instead.
|
|
2618
|
+
#
|
|
2619
|
+
# They are OPERATIONALLY AUTHORITATIVE only inside a validated current
|
|
2620
|
+
# generation. The journal remains ultimate truth: every rebuild, every
|
|
2621
|
+
# stale-generation fallback, Model-A versus harvest emission, bootstrap
|
|
2622
|
+
# handling and correction-batch application still derive from journal
|
|
2623
|
+
# records. Durable selector state may ACCELERATE a validated generation
|
|
2624
|
+
# and may never SUPERSEDE retained truth.
|
|
2625
|
+
#
|
|
2626
|
+
# One row, enforced structurally — unlike `stats_publication_stamp`,
|
|
2627
|
+
# whose duplicate row is a state that must resolve INDETERMINATE and
|
|
2628
|
+
# therefore may not be made impossible.
|
|
2629
|
+
conn.execute(
|
|
2630
|
+
"CREATE TABLE IF NOT EXISTS journal_selector_state ("
|
|
2631
|
+
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
2632
|
+
"generation_record_path TEXT, "
|
|
2633
|
+
"generation_stamped_at_utc TEXT, "
|
|
2634
|
+
"covered_segment TEXT, "
|
|
2635
|
+
"covered_offset INTEGER, "
|
|
2636
|
+
"next_sequence INTEGER NOT NULL DEFAULT 0, "
|
|
2637
|
+
"selector_version INTEGER NOT NULL, "
|
|
2638
|
+
# `cutover_seen` distinguishes "no cutover op exists" from "the op
|
|
2639
|
+
# exists and recorded no account". A plain NULL cannot carry both
|
|
2640
|
+
# answers, and conflating them re-runs the whole-journal cutover
|
|
2641
|
+
# scan F20 exists to remove.
|
|
2642
|
+
"cutover_seen INTEGER NOT NULL DEFAULT 0, "
|
|
2643
|
+
"cutover_account_key TEXT)"
|
|
2644
|
+
)
|
|
2645
|
+
conn.execute(
|
|
2646
|
+
"CREATE TABLE IF NOT EXISTS journal_selector_batches ("
|
|
2647
|
+
"batch_id TEXT PRIMARY KEY, "
|
|
2648
|
+
"status TEXT NOT NULL "
|
|
2649
|
+
" CHECK (status IN ('begin_only','completed','tainted')), "
|
|
2650
|
+
"action_count INTEGER, "
|
|
2651
|
+
"action_set_hash TEXT, "
|
|
2652
|
+
"begin_segment TEXT, begin_offset INTEGER, "
|
|
2653
|
+
"earliest_commit_segment TEXT, earliest_commit_offset INTEGER)"
|
|
2654
|
+
)
|
|
2655
|
+
# One row per marker and per action. A digest alone is not enough: when
|
|
2656
|
+
# a batch completes, the selector rebuilds every action's canonical core
|
|
2657
|
+
# to derive `actual_actions_hash`, and a split cycle — begin and actions
|
|
2658
|
+
# in an earlier generation, commit in this tick — decides completion NOW
|
|
2659
|
+
# from cores captured THEN. `action_core_json` is therefore retained
|
|
2660
|
+
# while the batch is `begin_only` OR `tainted`, and dropped only on
|
|
2661
|
+
# `completed`; an early taint does not end a batch's record stream.
|
|
2662
|
+
conn.execute(
|
|
2663
|
+
"CREATE TABLE IF NOT EXISTS journal_selector_batch_records ("
|
|
2664
|
+
"batch_id TEXT NOT NULL, "
|
|
2665
|
+
"kind TEXT NOT NULL CHECK (kind IN ('marker','action')), "
|
|
2666
|
+
"key TEXT NOT NULL, "
|
|
2667
|
+
"record_digest TEXT NOT NULL, "
|
|
2668
|
+
"identity_digest TEXT, "
|
|
2669
|
+
"sequence INTEGER NOT NULL, "
|
|
2670
|
+
"action_core_json TEXT, "
|
|
2671
|
+
"PRIMARY KEY (batch_id, kind, key))"
|
|
2672
|
+
)
|
|
2673
|
+
# Set only by Stage 3, reserved here so no stage depends on a table a
|
|
2674
|
+
# later stage creates (#496 S5b §4.7). The stats quota projection is
|
|
2675
|
+
# materialized FROM cache.db, so a partial cache recovery publishes a
|
|
2676
|
+
# semantically partial projection inside the generation; the flag is the
|
|
2677
|
+
# per-transaction gate that keeps that projection from being served. The
|
|
2678
|
+
# target is VERSIONED, not a bare coordinate, so a target written by one
|
|
2679
|
+
# binary is never misread by another.
|
|
2680
|
+
conn.execute(
|
|
2681
|
+
"CREATE TABLE IF NOT EXISTS stats_quota_projection_state ("
|
|
2682
|
+
"id INTEGER PRIMARY KEY CHECK (id = 1), "
|
|
2683
|
+
"incomplete INTEGER NOT NULL DEFAULT 0, "
|
|
2684
|
+
"target_version INTEGER NOT NULL DEFAULT 0, "
|
|
2685
|
+
"recovery_target_json TEXT)"
|
|
2509
2686
|
)
|
|
2510
2687
|
# In-place publication identity (#496 S3 §5). An in-place publish
|
|
2511
2688
|
# attaches the scratch read-only and detaches it, so the scratch
|
|
@@ -324,6 +324,40 @@ def _cctally():
|
|
|
324
324
|
return sys.modules["cctally"]
|
|
325
325
|
|
|
326
326
|
|
|
327
|
+
class _QuotaProjectionIncompleteUnavailable(BaseException):
|
|
328
|
+
"""The unmatchable stand-in `_quota_projection_incomplete_cls` falls back to.
|
|
329
|
+
|
|
330
|
+
A `BaseException` subclass nothing raises, so an `except` clause given this
|
|
331
|
+
class matches no exception and the handler below it runs unchanged.
|
|
332
|
+
"""
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _quota_projection_incomplete_cls():
|
|
336
|
+
"""`QuotaProjectionIncomplete`, resolved at call time (#496 S5b).
|
|
337
|
+
|
|
338
|
+
Call-time for the same reason `assert_projection_readable` is imported
|
|
339
|
+
inside its handlers: this module reaches `_cctally_quota` lazily, and an
|
|
340
|
+
`except` clause is evaluated only when an exception is being handled, so a
|
|
341
|
+
module-level import would buy nothing and add an import-order constraint.
|
|
342
|
+
|
|
343
|
+
Resolving it is therefore an import evaluated WHILE another exception is
|
|
344
|
+
already propagating, and an `ImportError` raised there would replace the
|
|
345
|
+
original exception and skip the generic handler that owns it. Both failure
|
|
346
|
+
modes answer with an unmatchable sentinel instead, so a resolution failure
|
|
347
|
+
degrades to the generic handler rather than to a different exception. The
|
|
348
|
+
class is deliberately NOT cached across calls: `_cctally_quota` is
|
|
349
|
+
re-imported into a fresh namespace by the test harness, and a cached class
|
|
350
|
+
object from an earlier namespace would stop matching the exception the
|
|
351
|
+
current one raises.
|
|
352
|
+
"""
|
|
353
|
+
try:
|
|
354
|
+
import _cctally_quota
|
|
355
|
+
|
|
356
|
+
return _cctally_quota.QuotaProjectionIncomplete
|
|
357
|
+
except (ImportError, AttributeError):
|
|
358
|
+
return _QuotaProjectionIncompleteUnavailable
|
|
359
|
+
|
|
360
|
+
|
|
327
361
|
# === Honest imports from extracted homes ===================================
|
|
328
362
|
# Spec 2026-05-17-cctally-core-kernel-extraction.md §3.3: kernel symbols
|
|
329
363
|
# import from _cctally_core; already-decentralized buckets (X = _lib_*,
|
|
@@ -698,7 +732,10 @@ def _build_codex_project_detail(context, qualified, observations, *, key: str) -
|
|
|
698
732
|
|
|
699
733
|
def _build_codex_block_detail(context, observations, *, key: str) -> dict[str, Any]:
|
|
700
734
|
from _lib_quota import build_blocks, forecast_quota, percent_milestones, quota_freshness
|
|
735
|
+
from _cctally_quota import assert_projection_readable
|
|
701
736
|
|
|
737
|
+
# BEFORE the SQL, per #496 S5b section 4.7.
|
|
738
|
+
assert_projection_readable(context.stats_conn)
|
|
702
739
|
rows = context.stats_conn.execute(
|
|
703
740
|
"SELECT source_root_key, logical_limit_key, observed_slot, window_minutes, "
|
|
704
741
|
"limit_name, resets_at_utc, current_percent, orphaned_at "
|
|
@@ -2942,7 +2979,8 @@ def _dashboard_build_daily_panel(conn: "sqlite3.Connection",
|
|
|
2942
2979
|
# resolves to `foo (parent_dir)`).
|
|
2943
2980
|
# - ``bucket_path`` = canonical equality key (``ProjectKey.bucket_path``)
|
|
2944
2981
|
# — the absolute on-disk path. Privacy-sensitive;
|
|
2945
|
-
# _lib_share.
|
|
2982
|
+
# _lib_share.render()/compose() prepare it away on
|
|
2983
|
+
# the share path.
|
|
2946
2984
|
|
|
2947
2985
|
# Per-tick memo (spec §6.4 + memory: *Pre-probe before sync_cache*).
|
|
2948
2986
|
# Keyed on (max(session_entries.id), current_week.week_start_at,
|
|
@@ -6320,6 +6358,17 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6320
6358
|
"error": "source capability unavailable",
|
|
6321
6359
|
})
|
|
6322
6360
|
return
|
|
6361
|
+
except _quota_projection_incomplete_cls():
|
|
6362
|
+
# #496 S5b, same reason as the `/api/milestones` branch: this route
|
|
6363
|
+
# reaches `_build_codex_block_detail`, whose gate runs before its
|
|
6364
|
+
# SQL, and the generic 400 below reported "capability unavailable"
|
|
6365
|
+
# for a reconcilable state and named no remedy.
|
|
6366
|
+
self._respond_json(503, {
|
|
6367
|
+
"code": "quota_projection_incomplete",
|
|
6368
|
+
"error": "quota view reconciling",
|
|
6369
|
+
"action": "cctally cache-sync",
|
|
6370
|
+
})
|
|
6371
|
+
return
|
|
6323
6372
|
except Exception as exc: # noqa: BLE001 — detailed diagnostics stay server-only.
|
|
6324
6373
|
self.log_error("/api/source/%s/%s failed: %r", source, resource, exc)
|
|
6325
6374
|
self._respond_json(400, {
|
|
@@ -6646,6 +6695,11 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6646
6695
|
# gets the same BEGIN/commit envelope as the stats conn.
|
|
6647
6696
|
stats_conn.execute("BEGIN")
|
|
6648
6697
|
cache_conn.execute("BEGIN")
|
|
6698
|
+
# The gate acquires no lock, so it is safe INSIDE this
|
|
6699
|
+
# transaction — and inside is the only placement that covers a
|
|
6700
|
+
# connection opened before the publication (#496 S5b section 4.7).
|
|
6701
|
+
from _cctally_quota import assert_projection_readable
|
|
6702
|
+
assert_projection_readable(stats_conn)
|
|
6649
6703
|
roots = tuple(sorted({
|
|
6650
6704
|
str(r[0]) for r in stats_conn.execute(
|
|
6651
6705
|
"SELECT DISTINCT source_root_key FROM quota_window_blocks "
|
|
@@ -6688,6 +6742,16 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6688
6742
|
})
|
|
6689
6743
|
return
|
|
6690
6744
|
self._send_milestones_json(200, {**result, "source": "codex", "key": key})
|
|
6745
|
+
except _quota_projection_incomplete_cls():
|
|
6746
|
+
# #496 S5b. A refused projection read is a RETRY signal over a
|
|
6747
|
+
# valid index, not a server fault, and the generic 500 below both
|
|
6748
|
+
# mislabels it and drops the one remedy the user can act on: the
|
|
6749
|
+
# message naming `cctally cache-sync` reached the server log only.
|
|
6750
|
+
self._send_milestones_json(503, {
|
|
6751
|
+
"error": "quota view reconciling",
|
|
6752
|
+
"code": "quota_projection_incomplete",
|
|
6753
|
+
"action": "cctally cache-sync",
|
|
6754
|
+
})
|
|
6691
6755
|
except Exception as exc: # noqa: BLE001
|
|
6692
6756
|
self.log_error("/api/milestones failed: %r", exc)
|
|
6693
6757
|
self.send_error(500, "milestones detail failed")
|
|
@@ -7398,6 +7462,12 @@ def cmd_dashboard(args: argparse.Namespace) -> int:
|
|
|
7398
7462
|
# SIGUSR1 is unavailable (Windows).
|
|
7399
7463
|
_register_faulthandler_sigusr1()
|
|
7400
7464
|
|
|
7465
|
+
# #496 S5b §4.7: a long-lived server, and the surface on which a gated
|
|
7466
|
+
# quota projection is most visible, so it may resume that recovery. One
|
|
7467
|
+
# attempt per throttle interval — see
|
|
7468
|
+
# `_cctally_journal._PROJECTION_RECONCILE_RETRY_SECONDS`.
|
|
7469
|
+
_cctally_core.enable_quota_projection_reconciliation()
|
|
7470
|
+
|
|
7401
7471
|
# Spec §5.7: capture the un-mutated argv + PATH-resolved entrypoint
|
|
7402
7472
|
# at boot so the in-place ``execvp`` after a successful update
|
|
7403
7473
|
# re-enters the user-facing wrapper (npm Node shim → CCTALLY_PYTHON
|
|
@@ -874,7 +874,9 @@ def _sync_failure_envelope(
|
|
|
874
874
|
if not error:
|
|
875
875
|
return None
|
|
876
876
|
|
|
877
|
-
def attributed(
|
|
877
|
+
def attributed(
|
|
878
|
+
database: str, *, corruption: bool = False, leg: str | None = None,
|
|
879
|
+
) -> bool:
|
|
878
880
|
for item in attributions or ():
|
|
879
881
|
item_database = (
|
|
880
882
|
item.get("database") if isinstance(item, dict)
|
|
@@ -884,9 +886,13 @@ def _sync_failure_envelope(
|
|
|
884
886
|
item.get("corruption") if isinstance(item, dict)
|
|
885
887
|
else getattr(item, "corruption", False)
|
|
886
888
|
)
|
|
889
|
+
item_leg = (
|
|
890
|
+
item.get("leg") if isinstance(item, dict)
|
|
891
|
+
else getattr(item, "leg", None)
|
|
892
|
+
)
|
|
887
893
|
if item_database == database and (
|
|
888
894
|
not corruption or bool(item_corruption)
|
|
889
|
-
):
|
|
895
|
+
) and (leg is None or item_leg == leg):
|
|
890
896
|
return True
|
|
891
897
|
return False
|
|
892
898
|
|
|
@@ -909,6 +915,26 @@ def _sync_failure_envelope(
|
|
|
909
915
|
"action": None,
|
|
910
916
|
}
|
|
911
917
|
|
|
918
|
+
# #496 S5b. A refused quota-projection read is the one attribution whose
|
|
919
|
+
# remedy the user can act on immediately, and without this branch it fell
|
|
920
|
+
# through to the terminal default below: `database="other"` matches nothing
|
|
921
|
+
# above it, so the chip read "⚠ server sync error" with no action, while
|
|
922
|
+
# the message naming `cctally cache-sync` sat in `last_sync_error`, which
|
|
923
|
+
# this module's own contract forbids putting in a visible text, title or
|
|
924
|
+
# aria surface. Keyed on the leg rather than on the raw text, because
|
|
925
|
+
# `database="other"` is deliberately generic and text classification is
|
|
926
|
+
# what the attribution vocabulary exists to replace.
|
|
927
|
+
if attributed("other", leg="quota-projection"):
|
|
928
|
+
return {
|
|
929
|
+
"kind": "quota_projection_incomplete",
|
|
930
|
+
"label": "quota view reconciling",
|
|
931
|
+
"detail": (
|
|
932
|
+
"The quota view is being rebuilt after an interrupted cache "
|
|
933
|
+
"recovery."
|
|
934
|
+
),
|
|
935
|
+
"action": "cctally cache-sync",
|
|
936
|
+
}
|
|
937
|
+
|
|
912
938
|
# conversations.db is intentionally outside core dashboard generations.
|
|
913
939
|
# Typed transcript ownership therefore degrades through the generic server
|
|
914
940
|
# contract: Doctor owns integrity diagnosis, and raw SQLite wording must
|
|
@@ -321,7 +321,8 @@ def _share_apply_content_toggles(snap_built, options: dict):
|
|
|
321
321
|
The render kernel consumes whatever the template builder emits, so
|
|
322
322
|
chart/table on-off can't be expressed by the builder alone (every
|
|
323
323
|
builder unconditionally emits both). Apply the toggle here, after
|
|
324
|
-
the builder, before `
|
|
324
|
+
the builder, before `render()` / `compose()` prepare it.
|
|
325
|
+
ShareSnapshot is frozen;
|
|
325
326
|
`dataclasses.replace` returns a new instance.
|
|
326
327
|
|
|
327
328
|
Defaults preserve pre-toggle behavior: `show_chart` defaults to
|
|
@@ -379,8 +380,11 @@ def _share_top_projects_for_range(
|
|
|
379
380
|
week, current 5h block).
|
|
380
381
|
|
|
381
382
|
NULL `project_path` collapses to the `(unknown)` sentinel. Anon
|
|
382
|
-
happens later in `
|
|
383
|
-
the
|
|
383
|
+
happens later, in the preparation pass `render()` and `compose()` run
|
|
384
|
+
over the raw snapshot; builders always emit real names per the
|
|
385
|
+
kernel's privacy chokepoint contract. (It is NOT `_scrub()`: that
|
|
386
|
+
function is retained for backward compatibility and no production
|
|
387
|
+
path calls it.)
|
|
384
388
|
"""
|
|
385
389
|
bucket: dict[str, float] = {}
|
|
386
390
|
try:
|
|
@@ -1114,9 +1118,11 @@ def _build_projects_share_panel_data(options: dict,
|
|
|
1114
1118
|
}
|
|
1115
1119
|
|
|
1116
1120
|
The Privacy invariant per spec §7.4 lives at the share-render gate
|
|
1117
|
-
(`_lib_share.
|
|
1118
|
-
display_keys + bucket_paths;
|
|
1119
|
-
when
|
|
1121
|
+
(`_lib_share.render()` / `compose()`), NOT here. This panel_data
|
|
1122
|
+
carries REAL display_keys + bucket_paths; the preparation pass those
|
|
1123
|
+
entry points run over the raw snapshot rewrites them when
|
|
1124
|
+
``reveal_projects=false``. (It is NOT `_scrub()`: that function is
|
|
1125
|
+
retained for backward compatibility and no production path calls it.)
|
|
1120
1126
|
"""
|
|
1121
1127
|
env: dict = getattr(snap, "projects_envelope", None) or {} if snap else {}
|
|
1122
1128
|
if not env:
|
|
@@ -1823,7 +1829,24 @@ class _SharePeriodError(ValueError):
|
|
|
1823
1829
|
|
|
1824
1830
|
def _share_public_failure(handler, exc: Exception, *, phase: str,
|
|
1825
1831
|
capability: bool = False) -> None:
|
|
1826
|
-
|
|
1832
|
+
# A privacy refusal is logged by CLASS ONLY (#503 S1 R10). Since the
|
|
1833
|
+
# refusal message widened to name the matched value, a `%r` of the
|
|
1834
|
+
# exception can put an absolute path, a UUID or an email address into the
|
|
1835
|
+
# dashboard log — and a log is a plausible thing to paste into a bug
|
|
1836
|
+
# report. The data is the user's own and the HTTP response below is
|
|
1837
|
+
# generic either way, so nothing reaches a remote client; this is about
|
|
1838
|
+
# what the log file accumulates. Every raise site in `_lib_share` sets
|
|
1839
|
+
# `classes`, and `SharePrivacyViolation` defaults it to a non-empty
|
|
1840
|
+
# sentinel, so this branch cannot fall through to the `%r` for a privacy
|
|
1841
|
+
# refusal even if a future raise site forgets the keyword. Any OTHER
|
|
1842
|
+
# exception is still logged in full, because its repr is the only
|
|
1843
|
+
# diagnostic there is.
|
|
1844
|
+
classes = getattr(exc, "classes", None)
|
|
1845
|
+
if classes:
|
|
1846
|
+
handler.log_error("/api/share/%s failed: %s: %s", phase,
|
|
1847
|
+
type(exc).__name__, ", ".join(classes))
|
|
1848
|
+
else:
|
|
1849
|
+
handler.log_error("/api/share/%s failed: %r", phase, exc)
|
|
1827
1850
|
if capability:
|
|
1828
1851
|
handler._respond_json(400, {
|
|
1829
1852
|
"code": "source_capability_unavailable",
|
|
@@ -1877,9 +1900,10 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
1877
1900
|
template_id against the registry, dispatches to the per-panel
|
|
1878
1901
|
`_build_<panel>_share_panel_data` helper to assemble the
|
|
1879
1902
|
builder-shaped dict from the current dashboard snapshot, runs the
|
|
1880
|
-
template's builder,
|
|
1881
|
-
|
|
1882
|
-
|
|
1903
|
+
template's builder, then renders via `_lib_share.render`, which
|
|
1904
|
+
prepares the RAW snapshot it is handed — anonymizing project labels
|
|
1905
|
+
when ``options.reveal_projects`` is False. Response:
|
|
1906
|
+
``{body, content_type, snapshot}``
|
|
1883
1907
|
where `snapshot` carries `kernel_version` + `data_digest` for the
|
|
1884
1908
|
v2 composer's drift detection (spec §5.2).
|
|
1885
1909
|
|
|
@@ -2024,11 +2048,16 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
2024
2048
|
source_snaps = tuple(
|
|
2025
2049
|
_share_apply_content_toggles(item, options) for item in source_snaps
|
|
2026
2050
|
)
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
|
|
2051
|
+
# FAIL CLOSED (#503 S1 F3). HTTP genuinely has an absent-field case, so
|
|
2052
|
+
# a default belongs here; it must resolve to anonymize, matching
|
|
2053
|
+
# /api/share/compose, which already defaulted closed. The kernel itself
|
|
2054
|
+
# has no default at all, so a fourth site cannot get this wrong.
|
|
2055
|
+
reveal = bool(options.get("reveal_projects", False))
|
|
2056
|
+
# No pre-scrub: the kernel's `render()` / `compose()` own the privacy
|
|
2057
|
+
# contract and require RAW snapshots (#503 S1). Pre-scrubbing here
|
|
2058
|
+
# renumbers aliases on the legacy path, and in the `source=all` branch it
|
|
2059
|
+
# merges two distinct projects that each mapped locally to `project-1`
|
|
2060
|
+
# into a single alias.
|
|
2032
2061
|
try:
|
|
2033
2062
|
if source == "all":
|
|
2034
2063
|
body = ls.compose(
|
|
@@ -2049,6 +2078,7 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
2049
2078
|
format=fmt,
|
|
2050
2079
|
theme=options.get("theme", "light"),
|
|
2051
2080
|
branding=not options.get("no_branding", False),
|
|
2081
|
+
reveal_projects=reveal,
|
|
2052
2082
|
)
|
|
2053
2083
|
except Exception as exc:
|
|
2054
2084
|
_share_public_failure(handler, exc, phase="render kernel")
|
|
@@ -2093,9 +2123,31 @@ def _handle_share_render_post_impl(handler) -> None:
|
|
|
2093
2123
|
if account_label is not None:
|
|
2094
2124
|
account_meta["account_label"] = account_label
|
|
2095
2125
|
|
|
2126
|
+
# #503 S1 B1 — does this export contain project names at all?
|
|
2127
|
+
#
|
|
2128
|
+
# The share modal's status line said "Export will show real project names"
|
|
2129
|
+
# on every panel. Some renders produce artifacts that are byte-identical in
|
|
2130
|
+
# both privacy modes apart from the `anonymized:` frontmatter line, so on
|
|
2131
|
+
# those the line was making a false statement — and a warning users learn to
|
|
2132
|
+
# disregard on Forecast is one they may disregard on Projects. The client
|
|
2133
|
+
# renders a third, neutral state from this flag.
|
|
2134
|
+
#
|
|
2135
|
+
# Which renders those are is derived per render from the snapshot in hand,
|
|
2136
|
+
# never counted or listed: it depends on the data the panel actually holds,
|
|
2137
|
+
# and it varies within a single panel. Do not state a number here.
|
|
2138
|
+
#
|
|
2139
|
+
# ADDITIVE per `docs/cli-contract.md`: an optional key does not bump a
|
|
2140
|
+
# schema version, and consumers must tolerate unknown keys. Derived from
|
|
2141
|
+
# the same RAW snapshots the renderer was just handed, through the kernel's
|
|
2142
|
+
# `_map_project_display` enumeration — never from a panel list, which would
|
|
2143
|
+
# be a second source of truth and wrong at template granularity.
|
|
2144
|
+
has_project_names = any(
|
|
2145
|
+
ls.has_project_identities(item) for item in source_snaps)
|
|
2146
|
+
|
|
2096
2147
|
handler._respond_json(200, {
|
|
2097
2148
|
"body": body,
|
|
2098
2149
|
"content_type": content_type,
|
|
2150
|
+
"has_project_names": has_project_names,
|
|
2099
2151
|
"snapshot": {
|
|
2100
2152
|
"kernel_version": ls.KERNEL_VERSION,
|
|
2101
2153
|
"panel": panel,
|
|
@@ -2282,10 +2334,14 @@ def _handle_share_compose_post_impl(handler) -> None:
|
|
|
2282
2334
|
_share_apply_content_toggles(item, composite_opts)
|
|
2283
2335
|
for item in source_snaps
|
|
2284
2336
|
)
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2337
|
+
# No pre-scrub (#503 S1): `compose()` prepares every section itself,
|
|
2338
|
+
# under one merged alias namespace. Scrubbing per section here is what
|
|
2339
|
+
# made `project-1` denote a different project in each section.
|
|
2340
|
+
#
|
|
2341
|
+
# The digest below is unaffected by the removal: `_share_digest_input`
|
|
2342
|
+
# reads only `snapshot.period` off each snapshot, never a label, so it
|
|
2343
|
+
# stays byte-identical and no basket section spuriously reads
|
|
2344
|
+
# "Outdated".
|
|
2289
2345
|
|
|
2290
2346
|
# Defensive: digest is non-blocking metadata — fall back to
|
|
2291
2347
|
# "" on failure rather than 500-ing the whole compose
|
|
@@ -15,6 +15,7 @@ from zoneinfo import ZoneInfo
|
|
|
15
15
|
|
|
16
16
|
from _cctally_core import get_week_start_name
|
|
17
17
|
from _cctally_quota import (
|
|
18
|
+
assert_projection_readable,
|
|
18
19
|
codex_five_hour_percent_at_crossing,
|
|
19
20
|
codex_quota_breakdown,
|
|
20
21
|
codex_physical_mutation_seq,
|
|
@@ -586,6 +587,9 @@ def _codex_weekly_periods(
|
|
|
586
587
|
# so the sentinel needs no NULL branch here (unlike the cache tables).
|
|
587
588
|
account_predicate = "" if account_key is None else "AND account_key = ? "
|
|
588
589
|
account_params: tuple = () if account_key is None else (account_key,)
|
|
590
|
+
# BEFORE the `try`, per #496 S5b section 4.7: beneath that handler a denial
|
|
591
|
+
# would be rendered as empty data rather than as an error.
|
|
592
|
+
assert_projection_readable(stats_conn)
|
|
589
593
|
try:
|
|
590
594
|
rows = stats_conn.execute(
|
|
591
595
|
"SELECT source_root_key, account_key, logical_limit_key, limit_name, "
|
|
@@ -931,6 +935,12 @@ def codex_projection_coherence(
|
|
|
931
935
|
physical signature only after its stats transaction commits, and its cache
|
|
932
936
|
sequence must still match before presentation can use it.
|
|
933
937
|
"""
|
|
938
|
+
# BEFORE the `try`, per #496 S5b section 4.7, and for the same reason every
|
|
939
|
+
# other gated site places it there: the handler below catches
|
|
940
|
+
# `ValueError`/`TypeError` as well as `sqlite3.Error`, so a refusal raised
|
|
941
|
+
# inside it would be rendered as `ProjectionCoherence(False,
|
|
942
|
+
# "projection_read_failed")` — a degraded-data verdict instead of a denial.
|
|
943
|
+
assert_projection_readable(context.stats_conn)
|
|
934
944
|
try:
|
|
935
945
|
active_roots = tuple(sorted(
|
|
936
946
|
str(row[0]) for row in context.cache_conn.execute(
|
|
@@ -1516,6 +1526,7 @@ def _quota_wire(
|
|
|
1516
1526
|
"""
|
|
1517
1527
|
if cycle is None or now_utc is None:
|
|
1518
1528
|
return ()
|
|
1529
|
+
assert_projection_readable(stats_conn)
|
|
1519
1530
|
try:
|
|
1520
1531
|
rows = stats_conn.execute(
|
|
1521
1532
|
"SELECT source_root_key, logical_limit_key, observed_slot, window_minutes, "
|
|
@@ -3151,6 +3162,7 @@ def _codex_block_account_keys(
|
|
|
3151
3162
|
if not root_keys:
|
|
3152
3163
|
return set()
|
|
3153
3164
|
placeholders = ",".join("?" for _ in root_keys)
|
|
3165
|
+
assert_projection_readable(stats_conn)
|
|
3154
3166
|
try:
|
|
3155
3167
|
return {
|
|
3156
3168
|
str(row[0] or _lib_accounts.UNATTRIBUTED)
|