cctally 1.63.0 → 1.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +7 -0
- package/bin/_cctally_cache.py +60 -19
- package/bin/_cctally_config.py +68 -0
- package/bin/_cctally_core.py +29 -0
- package/bin/_cctally_dashboard.py +364 -38
- package/bin/_cctally_doctor.py +24 -0
- package/bin/_cctally_parser.py +61 -0
- package/bin/_cctally_setup.py +22 -0
- package/bin/_cctally_telemetry.py +362 -0
- package/bin/_cctally_tui.py +445 -236
- package/bin/_cctally_update.py +53 -9
- package/bin/_lib_conversation_query.py +349 -36
- package/bin/_lib_doctor.py +32 -0
- package/bin/_lib_perf.py +180 -0
- package/bin/_lib_transcript_access.py +23 -0
- package/bin/cctally +98 -0
- package/bin/cctally-npm-postinstall.js +4 -0
- package/dashboard/static/assets/index-CJTUCpKt.js +80 -0
- package/dashboard/static/assets/index-DQWNrIqu.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +3 -1
- package/dashboard/static/assets/index-0jzYm75p.css +0 -1
- package/dashboard/static/assets/index-D_Ylyqsf.js +0 -80
|
@@ -2600,13 +2600,35 @@ class SSEHub:
|
|
|
2600
2600
|
import queue as _queue
|
|
2601
2601
|
with self._lock:
|
|
2602
2602
|
self._last = snapshot
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2603
|
+
# Latest-wins coalescing (#278 §2.6): every published snapshot is a
|
|
2604
|
+
# COMPLETE state replacement, so a client only ever needs the
|
|
2605
|
+
# newest. On a full queue drop the STALE queued frame and enqueue
|
|
2606
|
+
# the newest, so a slow subscriber (e.g. one filled by A2's rapid
|
|
2607
|
+
# partial republishes) still converges to the final hydrating=false
|
|
2608
|
+
# frame instead of dropping it — and a lagging client jumps to the
|
|
2609
|
+
# current state rather than replaying stale frames.
|
|
2610
|
+
#
|
|
2611
|
+
# Held under the hub lock so concurrent producers (the sync tick +
|
|
2612
|
+
# the update-check thread) can't interleave a get/put on the same
|
|
2613
|
+
# queue. All ops are non-blocking (put_nowait / get_nowait), so
|
|
2614
|
+
# holding the lock never back-pressures the producer, and the SSE
|
|
2615
|
+
# consumer only ever get()s (never puts), so after we make room the
|
|
2616
|
+
# re-put cannot lose to it.
|
|
2617
|
+
for q in self._queues:
|
|
2618
|
+
try:
|
|
2619
|
+
q.put_nowait(snapshot)
|
|
2620
|
+
except _queue.Full:
|
|
2621
|
+
try:
|
|
2622
|
+
q.get_nowait() # discard the oldest, stale frame
|
|
2623
|
+
except _queue.Empty:
|
|
2624
|
+
pass
|
|
2625
|
+
try:
|
|
2626
|
+
q.put_nowait(snapshot)
|
|
2627
|
+
except _queue.Full:
|
|
2628
|
+
# Defensive: a consumer racing between our get and put
|
|
2629
|
+
# could only have removed items, so this is unreachable
|
|
2630
|
+
# under the lock — but never raise out of publish().
|
|
2631
|
+
pass
|
|
2610
2632
|
|
|
2611
2633
|
|
|
2612
2634
|
STATIC_DIR = pathlib.Path(__file__).resolve().parent.parent / "dashboard" / "static"
|
|
@@ -6205,6 +6227,11 @@ def snapshot_to_envelope(snap: "DataSnapshot", *,
|
|
|
6205
6227
|
|
|
6206
6228
|
return {
|
|
6207
6229
|
"envelope_version": 2,
|
|
6230
|
+
# #278 Theme A: single additive first-paint hydration latch. True only
|
|
6231
|
+
# on the cheap seed + A2's partial republishes (data still being
|
|
6232
|
+
# assembled); False on every complete/stable snapshot. ``getattr``
|
|
6233
|
+
# default keeps positionally-constructed fixture snapshots serializing.
|
|
6234
|
+
"hydrating": bool(getattr(snap, "hydrating", False)),
|
|
6208
6235
|
"generated_at": _iso_z(snap.generated_at),
|
|
6209
6236
|
# last_sync_at in DataSnapshot is a monotonic float, not a wall
|
|
6210
6237
|
# clock — the envelope's wall-clock moment is unknowable from
|
|
@@ -6567,6 +6594,112 @@ def _cached_file_sigs(conn, paths):
|
|
|
6567
6594
|
return out
|
|
6568
6595
|
|
|
6569
6596
|
|
|
6597
|
+
# ── /api/debug/backend on-demand cache-state helpers (issue #276, Session A) ──
|
|
6598
|
+
# All read-only, cheap, and privacy-safe: they leak ONLY row counts, signature
|
|
6599
|
+
# legs (ints/tuples), pending-flag names, and the tool version — never prompt /
|
|
6600
|
+
# prose / paths. Computed on demand so cache_state is available even with
|
|
6601
|
+
# tracing off (the phase tree, by contrast, is present only when traced).
|
|
6602
|
+
|
|
6603
|
+
# Safe cache-table names surfaced as `dataset` row counts (no content read).
|
|
6604
|
+
_DEBUG_CACHE_TABLES = (
|
|
6605
|
+
"session_entries",
|
|
6606
|
+
"session_files",
|
|
6607
|
+
"conversation_messages",
|
|
6608
|
+
"conversation_sessions",
|
|
6609
|
+
"conversation_ai_titles",
|
|
6610
|
+
"conversation_file_touches",
|
|
6611
|
+
"codex_session_entries",
|
|
6612
|
+
"codex_session_files",
|
|
6613
|
+
)
|
|
6614
|
+
|
|
6615
|
+
|
|
6616
|
+
def _debug_cache_table_counts(cache_conn) -> dict:
|
|
6617
|
+
"""Row counts per known cache table. Absent tables (partially-migrated /
|
|
6618
|
+
fresh cache) are omitted rather than erroring."""
|
|
6619
|
+
counts: dict = {}
|
|
6620
|
+
for table in _DEBUG_CACHE_TABLES:
|
|
6621
|
+
try:
|
|
6622
|
+
row = cache_conn.execute(
|
|
6623
|
+
f"SELECT COUNT(*) FROM {table}" # noqa: S608 — fixed allowlist
|
|
6624
|
+
).fetchone()
|
|
6625
|
+
counts[table] = int(row[0])
|
|
6626
|
+
except sqlite3.Error:
|
|
6627
|
+
pass
|
|
6628
|
+
return counts
|
|
6629
|
+
|
|
6630
|
+
|
|
6631
|
+
def _debug_cache_state(cache_conn) -> dict:
|
|
6632
|
+
"""On-demand signature legs + pending-reingest flags + generation.
|
|
6633
|
+
|
|
6634
|
+
The signature legs are the canonical ``compute_signature`` fields (ints /
|
|
6635
|
+
a small tuple). stats.db is opened READ-ONLY via a dispatcher-free URI
|
|
6636
|
+
connection so this diagnostic never forward-migrates or creates a DB; if it
|
|
6637
|
+
is absent the stats legs degrade (each ``_max_id`` / ``_reset_sig`` already
|
|
6638
|
+
returns 0 on a missing table)."""
|
|
6639
|
+
c = _cctally()
|
|
6640
|
+
sc = c._load_sibling("_lib_snapshot_cache")
|
|
6641
|
+
state: dict = {"generation": sc.current_generation()}
|
|
6642
|
+
stats_conn = None
|
|
6643
|
+
try:
|
|
6644
|
+
stats_conn = sqlite3.connect(
|
|
6645
|
+
f"{_cctally_core.DB_PATH.as_uri()}?mode=ro", uri=True
|
|
6646
|
+
)
|
|
6647
|
+
except sqlite3.Error:
|
|
6648
|
+
stats_conn = None
|
|
6649
|
+
try:
|
|
6650
|
+
if stats_conn is not None:
|
|
6651
|
+
sig = sc.compute_signature(
|
|
6652
|
+
cache_conn, stats_conn, generation=sc.current_generation()
|
|
6653
|
+
)
|
|
6654
|
+
state["signature"] = {
|
|
6655
|
+
"max_entry_id": sig.max_entry_id,
|
|
6656
|
+
"max_wus_id": sig.max_wus_id,
|
|
6657
|
+
"max_wcs_id": sig.max_wcs_id,
|
|
6658
|
+
"reset_sig": list(sig.reset_sig),
|
|
6659
|
+
"max_codex_id": sig.max_codex_id,
|
|
6660
|
+
"entry_mutation_seq": sig.entry_mutation_seq,
|
|
6661
|
+
}
|
|
6662
|
+
else:
|
|
6663
|
+
state["signature"] = {
|
|
6664
|
+
"max_entry_id": sc._max_id(cache_conn, "session_entries"),
|
|
6665
|
+
"entry_mutation_seq": sc._entry_mutation_seq(cache_conn),
|
|
6666
|
+
}
|
|
6667
|
+
except sqlite3.Error as exc:
|
|
6668
|
+
state["signature"] = {"_error": f"{type(exc).__name__}: {exc}"}
|
|
6669
|
+
finally:
|
|
6670
|
+
if stats_conn is not None:
|
|
6671
|
+
stats_conn.close()
|
|
6672
|
+
# Pending reingest / backfill flag NAMES (never values) set in cache_meta.
|
|
6673
|
+
try:
|
|
6674
|
+
rows = cache_conn.execute(
|
|
6675
|
+
"SELECT key FROM cache_meta "
|
|
6676
|
+
"WHERE key LIKE '%_pending' AND value IS NOT NULL"
|
|
6677
|
+
).fetchall()
|
|
6678
|
+
state["pending_reingest"] = sorted(r[0] for r in rows)
|
|
6679
|
+
except sqlite3.Error:
|
|
6680
|
+
state["pending_reingest"] = []
|
|
6681
|
+
# Walk-complete sentinel presence (bool only — no timestamp leak).
|
|
6682
|
+
try:
|
|
6683
|
+
state["walk_complete"] = cache_conn.execute(
|
|
6684
|
+
"SELECT 1 FROM cache_meta WHERE key='claude_ingest_walk_complete'"
|
|
6685
|
+
).fetchone() is not None
|
|
6686
|
+
except sqlite3.Error:
|
|
6687
|
+
state["walk_complete"] = False
|
|
6688
|
+
return state
|
|
6689
|
+
|
|
6690
|
+
|
|
6691
|
+
def _debug_tool_version() -> str:
|
|
6692
|
+
"""The running tool version (latest stamped CHANGELOG header), or
|
|
6693
|
+
``"unknown"``. Safe to expose (already on the public update surface)."""
|
|
6694
|
+
try:
|
|
6695
|
+
v = _cctally()._load_sibling(
|
|
6696
|
+
"_lib_changelog"
|
|
6697
|
+
)._read_latest_changelog_version()
|
|
6698
|
+
return v[0] if v else "unknown"
|
|
6699
|
+
except Exception: # noqa: BLE001 — diagnostic must never raise
|
|
6700
|
+
return "unknown"
|
|
6701
|
+
|
|
6702
|
+
|
|
6570
6703
|
class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
6571
6704
|
"""Routes:
|
|
6572
6705
|
GET / → dashboard.html
|
|
@@ -6693,42 +6826,58 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6693
6826
|
self._handle_share_history_get()
|
|
6694
6827
|
elif path == "/api/doctor":
|
|
6695
6828
|
self._handle_get_doctor()
|
|
6829
|
+
elif path == "/api/debug/backend":
|
|
6830
|
+
# Loopback-only backend diagnostic (issue #276): last-build phase
|
|
6831
|
+
# timings + on-demand cache-state. Its own stricter gate (never
|
|
6832
|
+
# expose_transcripts); see _handle_get_debug_backend.
|
|
6833
|
+
self._handle_get_debug_backend()
|
|
6696
6834
|
elif path == "/api/conversations/facets":
|
|
6697
|
-
self.
|
|
6835
|
+
with self._perf_scope("endpoint.conversations_facets"):
|
|
6836
|
+
self._handle_get_conversations_facets()
|
|
6698
6837
|
elif path == "/api/conversations":
|
|
6699
|
-
self.
|
|
6838
|
+
with self._perf_scope("endpoint.conversations"):
|
|
6839
|
+
self._handle_get_conversations()
|
|
6700
6840
|
elif path == "/api/conversation/search":
|
|
6701
|
-
self.
|
|
6841
|
+
with self._perf_scope("endpoint.conversation_search"):
|
|
6842
|
+
self._handle_get_conversation_search()
|
|
6702
6843
|
elif path.startswith("/api/conversation/") and path.endswith("/payload"):
|
|
6703
6844
|
# #178: on-demand load-full. Matched BEFORE the <id> reader
|
|
6704
6845
|
# catch-all (same precedence as /api/conversation/search).
|
|
6705
|
-
self.
|
|
6846
|
+
with self._perf_gate().phase("endpoint.conversation_payload"):
|
|
6847
|
+
self._handle_get_conversation_payload(path)
|
|
6706
6848
|
elif path.startswith("/api/conversation/") and path.endswith("/media"):
|
|
6707
6849
|
# #177 S4: on-demand media bytes. Matched BEFORE the <id> reader.
|
|
6708
|
-
self.
|
|
6850
|
+
with self._perf_gate().phase("endpoint.conversation_media"):
|
|
6851
|
+
self._handle_get_conversation_media(path)
|
|
6709
6852
|
elif path.startswith("/api/conversation/") and path.endswith("/outline"):
|
|
6710
6853
|
# #177 S5: full-session outline skeleton + stats. Matched BEFORE
|
|
6711
6854
|
# the <id> reader catch-all (Codex F2 — same precedence as /payload).
|
|
6712
|
-
self.
|
|
6855
|
+
with self._perf_scope("endpoint.conversation_outline"):
|
|
6856
|
+
self._handle_get_conversation_outline(path)
|
|
6713
6857
|
elif path.startswith("/api/conversation/") and path.endswith("/find"):
|
|
6714
6858
|
# #177 S6: in-conversation find → rendered-turn anchors. Matched
|
|
6715
6859
|
# BEFORE the <id> reader catch-all (same precedence as /outline).
|
|
6716
|
-
self.
|
|
6860
|
+
with self._perf_scope("endpoint.conversation_find"):
|
|
6861
|
+
self._handle_get_conversation_find(path)
|
|
6717
6862
|
elif path.startswith("/api/conversation/") and path.endswith("/events"):
|
|
6718
6863
|
# Live-tail SSE for the open reader (spec §2). Matched BEFORE the
|
|
6719
6864
|
# <id> reader catch-all.
|
|
6720
|
-
self.
|
|
6865
|
+
with self._perf_gate().phase("endpoint.conversation_events"):
|
|
6866
|
+
self._handle_get_conversation_events(path)
|
|
6721
6867
|
elif path.startswith("/api/conversation/") and path.endswith("/export"):
|
|
6722
6868
|
# #217 S5: whole-session Markdown export (F1/F5). Matched BEFORE the
|
|
6723
6869
|
# <id> reader catch-all (same precedence as /outline).
|
|
6724
|
-
self.
|
|
6870
|
+
with self._perf_scope("endpoint.conversation_export"):
|
|
6871
|
+
self._handle_get_conversation_export(path)
|
|
6725
6872
|
elif path.startswith("/api/conversation/") and path.endswith("/prompts"):
|
|
6726
6873
|
# #217 S7: ordered main-thread prompt spine for session comparison
|
|
6727
6874
|
# (F10). Matched BEFORE the <id> reader catch-all (same precedence
|
|
6728
6875
|
# as /outline).
|
|
6729
|
-
self.
|
|
6876
|
+
with self._perf_scope("endpoint.conversation_prompts"):
|
|
6877
|
+
self._handle_get_conversation_prompts(path)
|
|
6730
6878
|
elif path.startswith("/api/conversation/"):
|
|
6731
|
-
self.
|
|
6879
|
+
with self._perf_scope("endpoint.conversation_detail"):
|
|
6880
|
+
self._handle_get_conversation_detail(path)
|
|
6732
6881
|
else:
|
|
6733
6882
|
self.send_error(404, "not found")
|
|
6734
6883
|
|
|
@@ -6924,6 +7073,95 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
6924
7073
|
return False
|
|
6925
7074
|
return True
|
|
6926
7075
|
|
|
7076
|
+
@staticmethod
|
|
7077
|
+
def _perf_gate():
|
|
7078
|
+
"""Lazy-load the opt-in backend phase-instrumentation collector (#276).
|
|
7079
|
+
Near-noop when tracing is off (``phase()`` returns a shared singleton),
|
|
7080
|
+
so the coarse ``endpoint.*`` wraps in ``do_GET`` cost nothing by
|
|
7081
|
+
default."""
|
|
7082
|
+
return sys.modules["cctally"]._load_sibling("_lib_perf")
|
|
7083
|
+
|
|
7084
|
+
@contextlib.contextmanager
|
|
7085
|
+
def _perf_scope(self, name):
|
|
7086
|
+
"""Like ``_perf_gate().phase(name)`` but ALSO stashes the completed root
|
|
7087
|
+
so the loopback ``/api/debug/backend`` surface can read a CONVERSATION
|
|
7088
|
+
trace, not just the ``/api/data`` snapshot tree (Session C / M5 — the
|
|
7089
|
+
Session A dead-end: no ``stash_last`` ran on a request thread). On enter
|
|
7090
|
+
it ``reset_thread()``s (each conversation handler runs on its own request
|
|
7091
|
+
thread, so this can't clobber the snapshot-build thread's tree) and opens
|
|
7092
|
+
``phase(name)``; on clean exit it stashes ``current_root()``. Near-noop
|
|
7093
|
+
when ``CCTALLY_PERF_TRACE`` is off — ``phase()`` returns ``_NULL_PHASE``,
|
|
7094
|
+
no root is pushed, so ``current_root()`` is ``None`` and ``stash_last``
|
|
7095
|
+
early-returns. Used ONLY for the short-lived, assembly-relevant routes;
|
|
7096
|
+
the long-lived ``/events`` SSE keeps a plain, non-stashing wrap (Codex F3)
|
|
7097
|
+
so it can never overwrite the last useful assembly trace on disconnect."""
|
|
7098
|
+
perf = self._perf_gate()
|
|
7099
|
+
perf.reset_thread()
|
|
7100
|
+
with perf.phase(name):
|
|
7101
|
+
yield
|
|
7102
|
+
try:
|
|
7103
|
+
perf.stash_last(
|
|
7104
|
+
perf.current_root(),
|
|
7105
|
+
generated_at=dt.datetime.now(dt.timezone.utc).isoformat())
|
|
7106
|
+
except Exception: # noqa: BLE001 — a diagnostic stash must never 500
|
|
7107
|
+
pass
|
|
7108
|
+
|
|
7109
|
+
def _require_debug_backend_allowed(self) -> bool:
|
|
7110
|
+
"""Gate for ``/api/debug/backend`` (issue #276) — STRICTER than the
|
|
7111
|
+
transcript gate.
|
|
7112
|
+
|
|
7113
|
+
PRIMARY: the TCP peer (``client_address[0]``) must be loopback — the
|
|
7114
|
+
unspoofable signal (the dashboard can bind ``0.0.0.0``, so a
|
|
7115
|
+
``Host``-only check is spoofable). DEFENSE-IN-DEPTH: the ``Host``
|
|
7116
|
+
authority must ALSO be an IP-literal loopback (anti-DNS-rebinding).
|
|
7117
|
+
``expose_transcripts`` is NEVER consulted. 403 + ``False`` otherwise.
|
|
7118
|
+
"""
|
|
7119
|
+
ta = self._transcript_gate()
|
|
7120
|
+
peer = self.client_address[0] if self.client_address else ""
|
|
7121
|
+
host = self.headers.get("Host")
|
|
7122
|
+
if not ta.debug_backend_allowed(peer, host):
|
|
7123
|
+
self._respond_403("forbidden")
|
|
7124
|
+
return False
|
|
7125
|
+
return True
|
|
7126
|
+
|
|
7127
|
+
def _handle_get_debug_backend(self) -> None:
|
|
7128
|
+
"""GET ``/api/debug/backend`` — loopback-only backend diagnostic (#276).
|
|
7129
|
+
|
|
7130
|
+
Returns the last completed snapshot build's phase-timing tree (present
|
|
7131
|
+
only if the dashboard was started with ``CCTALLY_PERF_TRACE=1``; else
|
|
7132
|
+
``null`` + a ``tracing_disabled`` note) plus on-demand cache-table row
|
|
7133
|
+
counts and signature legs. Leaks no prompt / prose / paths — timings,
|
|
7134
|
+
counts, flag names, and safe cache-table names only. ``schemaVersion``
|
|
7135
|
+
is 1 but the surface is documented UNSTABLE (a diagnostic, not a
|
|
7136
|
+
consumer contract): phase names / nesting / fields may change without a
|
|
7137
|
+
version bump.
|
|
7138
|
+
"""
|
|
7139
|
+
if not self._require_debug_backend_allowed():
|
|
7140
|
+
return
|
|
7141
|
+
last = self._perf_gate().last_backend_perf()
|
|
7142
|
+
dataset: dict = {}
|
|
7143
|
+
cache_state: dict = {}
|
|
7144
|
+
try:
|
|
7145
|
+
conn = open_cache_db()
|
|
7146
|
+
try:
|
|
7147
|
+
dataset = _debug_cache_table_counts(conn)
|
|
7148
|
+
cache_state = _debug_cache_state(conn)
|
|
7149
|
+
finally:
|
|
7150
|
+
conn.close()
|
|
7151
|
+
except Exception as exc: # noqa: BLE001 — a diagnostic must not 500 loudly
|
|
7152
|
+
cache_state = {"_error": f"{type(exc).__name__}: {exc}"}
|
|
7153
|
+
body = {
|
|
7154
|
+
"schemaVersion": 1,
|
|
7155
|
+
"version": _debug_tool_version(),
|
|
7156
|
+
"generated_at": (last or {}).get("generated_at"),
|
|
7157
|
+
"dataset": dataset,
|
|
7158
|
+
"phases": (last or {}).get("phases"),
|
|
7159
|
+
"cache_state": cache_state,
|
|
7160
|
+
}
|
|
7161
|
+
if body["phases"] is None:
|
|
7162
|
+
body["note"] = "tracing_disabled"
|
|
7163
|
+
self._respond_json(200, body)
|
|
7164
|
+
|
|
6927
7165
|
def _handle_post_settings(self) -> None:
|
|
6928
7166
|
"""Persist a settings update and trigger an immediate SSE broadcast.
|
|
6929
7167
|
|
|
@@ -8765,13 +9003,14 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
8765
9003
|
on bad input). On success returns a dict of ``list_conversations``
|
|
8766
9004
|
kwargs: ``date_from``/``date_to`` (UTC-ISO bounds), ``projects``
|
|
8767
9005
|
(list[str] | None), ``cost_min``/``cost_max`` (float | None),
|
|
8768
|
-
``rebuild_min`` (int | None)
|
|
9006
|
+
``rebuild_min`` (int | None), ``models`` (list[str] | None — the #278
|
|
9007
|
+
Theme C model-family axis). Empty/blank params drop to ``None``.
|
|
8769
9008
|
|
|
8770
9009
|
Numeric axes validate strictly (a non-numeric cost / non-integer
|
|
8771
9010
|
rebuild threshold is a hard 400). Date bounds route through the pure
|
|
8772
9011
|
``_lib_dashboard_dates.parse_filter_date_range`` helper, which resolves
|
|
8773
9012
|
naive date-only bounds in ``display.tz`` and raises ``ValueError`` (→
|
|
8774
|
-
400) on a malformed date. Projects accept BOTH repeated
|
|
9013
|
+
400) on a malformed date. Projects AND models accept BOTH repeated
|
|
8775
9014
|
``?projects=a&projects=b`` and a single comma-joined ``?projects=a,b``.
|
|
8776
9015
|
"""
|
|
8777
9016
|
def _float(name):
|
|
@@ -8806,6 +9045,15 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
8806
9045
|
if projects and len(projects) == 1 and "," in projects[0]:
|
|
8807
9046
|
projects = [s for s in projects[0].split(",") if s] or None
|
|
8808
9047
|
|
|
9048
|
+
# #278 Theme C: the model-family axis, mirroring projects. Accepts both
|
|
9049
|
+
# repeated ?models=opus&models=sonnet and a single comma-joined
|
|
9050
|
+
# ?models=opus,sonnet; blank/empty -> None. No numeric validation (enum-ish
|
|
9051
|
+
# strings); an unknown/typo'd family is a PRESENT axis that resolves to
|
|
9052
|
+
# zero ids in _model_clause -> zero results, never a silent unrestrict.
|
|
9053
|
+
models = [m for m in q.get("models", []) if m] or None
|
|
9054
|
+
if models and len(models) == 1 and "," in models[0]:
|
|
9055
|
+
models = [s for s in models[0].split(",") if s] or None
|
|
9056
|
+
|
|
8809
9057
|
date_from = _qs_str(q, "date_from", "") or None
|
|
8810
9058
|
date_to = _qs_str(q, "date_to", "") or None
|
|
8811
9059
|
if date_from or date_to:
|
|
@@ -8832,6 +9080,7 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
8832
9080
|
"cost_min": cost_min,
|
|
8833
9081
|
"cost_max": cost_max,
|
|
8834
9082
|
"rebuild_min": rebuild_min,
|
|
9083
|
+
"models": models,
|
|
8835
9084
|
}
|
|
8836
9085
|
|
|
8837
9086
|
def _handle_get_conversations(self) -> None:
|
|
@@ -8972,6 +9221,14 @@ class DashboardHTTPHandler(BaseHTTPRequestHandler):
|
|
|
8972
9221
|
self.wfile.write(b": keep-alive\n\n")
|
|
8973
9222
|
self.wfile.flush()
|
|
8974
9223
|
|
|
9224
|
+
# #278 Theme B: signal that this connection is ACTIVELY live-tailing
|
|
9225
|
+
# (not degraded to keep-alive). The client sets `live` only on this, so
|
|
9226
|
+
# passive/cache-open-failed streams fall back to the memo-backed global
|
|
9227
|
+
# tick instead of stranding updates (Codex F4). Passive branch above
|
|
9228
|
+
# emits only ': keep-alive', so 'ready' is unambiguous.
|
|
9229
|
+
self.wfile.write(b"event: ready\ndata: {}\n\n")
|
|
9230
|
+
self.wfile.flush()
|
|
9231
|
+
|
|
8975
9232
|
files = _resolve()
|
|
8976
9233
|
# Best-effort connect ingest for immediacy, then baseline `seen`
|
|
8977
9234
|
# from the cache's own offsets (session_files) so any pre-connect
|
|
@@ -9824,24 +10081,93 @@ def _dashboard_wait_for_signal(
|
|
|
9824
10081
|
|
|
9825
10082
|
|
|
9826
10083
|
def _dashboard_initial_snapshot(args, *, pinned_now, display_tz_pref_override):
|
|
9827
|
-
"""#
|
|
9828
|
-
HTTP port binds
|
|
9829
|
-
|
|
9830
|
-
|
|
9831
|
-
|
|
9832
|
-
|
|
9833
|
-
|
|
9834
|
-
|
|
9835
|
-
|
|
9836
|
-
|
|
9837
|
-
|
|
9838
|
-
|
|
9839
|
-
|
|
9840
|
-
|
|
9841
|
-
|
|
9842
|
-
|
|
9843
|
-
|
|
9844
|
-
|
|
10084
|
+
"""#278 Theme A (A1): build the dashboard's first snapshot as a CHEAP
|
|
10085
|
+
partial on a normal launch so the HTTP port binds in ~110ms instead of
|
|
10086
|
+
waiting on the ~2.2s full aggregation. #179 already deferred the *ingest*
|
|
10087
|
+
half of cold start (the background ``_DashboardSyncThread`` owns
|
|
10088
|
+
``sync_cache``); this defers the *aggregation* half too.
|
|
10089
|
+
|
|
10090
|
+
Normal launch (``not args.no_sync``): populate only the two sub-ms headline
|
|
10091
|
+
panels — ``current_week`` + ``forecast`` — plus the real doctor +
|
|
10092
|
+
envelope-config precompute, and set ``hydrating=True``; every heavy panel
|
|
10093
|
+
stays at its empty default. The background thread's first tick runs the
|
|
10094
|
+
full cold build + SSE-publish, and the client hydrates the heavy panels
|
|
10095
|
+
from that frame. Built via the INDIVIDUAL builder helpers, NOT
|
|
10096
|
+
``_tui_build_snapshot`` — so it never calls ``store_dispatch_state`` and
|
|
10097
|
+
never poisons the idle memo / accelerator caches (§1.1): the dispatch memo
|
|
10098
|
+
stays empty, so the first background tick sees ``prior_key=None`` →
|
|
10099
|
+
non-idle → a full cold build, and idle-reuse can never serve the partial.
|
|
10100
|
+
``_tui_build_current_week`` / ``_tui_build_forecast_view`` touch no
|
|
10101
|
+
process-local cache state with their default args (Codex P3), so the seed
|
|
10102
|
+
leaves every accelerator pristine.
|
|
10103
|
+
|
|
10104
|
+
``--no-sync`` (§1.2): there is no background thread to fill the partial in,
|
|
10105
|
+
so the cheap seed would become the PERMANENT state (heavy panels never
|
|
10106
|
+
populate). Keep the full pre-bind build (``hydrating=False``) — a ~2s bind
|
|
10107
|
+
in niche frozen-data mode is acceptable since nothing hydrates later.
|
|
10108
|
+
|
|
10109
|
+
§1.3: ``snapshot_to_envelope`` runs the REAL doctor inline PER CONNECTION
|
|
10110
|
+
when ``doctor_payload is None`` and KeyErrors on an ``envelope_precompute``
|
|
10111
|
+
missing ``"config"`` / the update fields, so a None-doctor seed would be
|
|
10112
|
+
WRONG (worse: every SSE client's first serialization would re-run doctor).
|
|
10113
|
+
The seed therefore runs BOTH precomputes for real (~110ms total). ``getattr``
|
|
10114
|
+
so minimal test ``args`` namespaces (no ``host`` / ``no_sync``) still build.
|
|
10115
|
+
"""
|
|
10116
|
+
c = _cctally()
|
|
10117
|
+
tui = c._cctally_tui
|
|
10118
|
+
if getattr(args, "no_sync", False):
|
|
10119
|
+
# Route _tui_build_snapshot through the cctally module (its re-export)
|
|
10120
|
+
# so ``monkeypatch.setitem(ns, "_tui_build_snapshot", spy)`` in tests
|
|
10121
|
+
# propagates — identical to the pre-change call form.
|
|
10122
|
+
return c._tui_build_snapshot(
|
|
10123
|
+
now_utc=pinned_now, skip_sync=True,
|
|
10124
|
+
display_tz_pref_override=display_tz_pref_override,
|
|
10125
|
+
precompute_envelope=True, runtime_bind=getattr(args, "host", None),
|
|
10126
|
+
)
|
|
10127
|
+
|
|
10128
|
+
import time as _time
|
|
10129
|
+
now_utc = pinned_now or dt.datetime.now(dt.timezone.utc)
|
|
10130
|
+
runtime_bind = getattr(args, "host", None)
|
|
10131
|
+
base = tui._tui_empty_snapshot(now_utc)
|
|
10132
|
+
errors: list[str] = []
|
|
10133
|
+
cw = None
|
|
10134
|
+
fc = None
|
|
10135
|
+
fc_view = None
|
|
10136
|
+
conn = open_db()
|
|
10137
|
+
try:
|
|
10138
|
+
try:
|
|
10139
|
+
cw = tui._tui_build_current_week(conn, now_utc, skip_sync=True)
|
|
10140
|
+
except Exception as exc: # noqa: BLE001 — never block the bind
|
|
10141
|
+
errors.append(f"current-week: {exc}")
|
|
10142
|
+
try:
|
|
10143
|
+
fc_view = tui._tui_build_forecast_view(conn, now_utc, skip_sync=True)
|
|
10144
|
+
fc = fc_view.output if fc_view is not None else None
|
|
10145
|
+
except Exception as exc: # noqa: BLE001
|
|
10146
|
+
errors.append(f"forecast: {exc}")
|
|
10147
|
+
finally:
|
|
10148
|
+
conn.close()
|
|
10149
|
+
# §1.3: run BOTH precomputes for real so the envelope serializes cleanly
|
|
10150
|
+
# without the per-connection inline-doctor fork or the config/update KeyErrors.
|
|
10151
|
+
doctor_payload = None
|
|
10152
|
+
envelope_precompute = None
|
|
10153
|
+
try:
|
|
10154
|
+
envelope_precompute = tui._tui_precompute_envelope_config(load_config())
|
|
10155
|
+
except Exception as exc: # noqa: BLE001
|
|
10156
|
+
errors.append(f"envelope-precompute: {exc}")
|
|
10157
|
+
try:
|
|
10158
|
+
doctor_payload = tui._tui_precompute_doctor_payload(now_utc, runtime_bind)
|
|
10159
|
+
except Exception as exc: # noqa: BLE001
|
|
10160
|
+
errors.append(f"doctor-precompute: {exc}")
|
|
10161
|
+
return dataclasses.replace(
|
|
10162
|
+
base,
|
|
10163
|
+
current_week=cw,
|
|
10164
|
+
forecast=fc,
|
|
10165
|
+
forecast_view=fc_view,
|
|
10166
|
+
last_sync_at=_time.monotonic(),
|
|
10167
|
+
last_sync_error=("; ".join(errors) if errors else None),
|
|
10168
|
+
doctor_payload=doctor_payload,
|
|
10169
|
+
envelope_precompute=envelope_precompute,
|
|
10170
|
+
hydrating=True,
|
|
9845
10171
|
)
|
|
9846
10172
|
|
|
9847
10173
|
|
package/bin/_cctally_doctor.py
CHANGED
|
@@ -416,6 +416,26 @@ def doctor_gather_state(
|
|
|
416
416
|
except (json.JSONDecodeError, OSError):
|
|
417
417
|
pass
|
|
418
418
|
|
|
419
|
+
# ── Telemetry (anonymous install-count, spec 2026-07-07) ─────────
|
|
420
|
+
# Resolve the opt-out state via the pure kernel predicate — it reads env
|
|
421
|
+
# + config + the dev-checkout fact and NEVER mints an install_id / touches
|
|
422
|
+
# any marker (read-only H1 invariant). Uses the same raw config read as the
|
|
423
|
+
# safety block so doctor never auto-creates config.json; a missing/corrupt
|
|
424
|
+
# config degrades to `{}` (env/dev precedence still resolves correctly).
|
|
425
|
+
telemetry_enabled = True
|
|
426
|
+
telemetry_reason = "enabled"
|
|
427
|
+
try:
|
|
428
|
+
raw_tele_cfg: dict = {}
|
|
429
|
+
if _cctally_core.CONFIG_PATH.exists():
|
|
430
|
+
loaded = json.loads(_cctally_core.CONFIG_PATH.read_text(encoding="utf-8"))
|
|
431
|
+
if isinstance(loaded, dict):
|
|
432
|
+
raw_tele_cfg = loaded
|
|
433
|
+
telemetry_enabled, telemetry_reason = c.resolve_telemetry_state(raw_tele_cfg)
|
|
434
|
+
except Exception:
|
|
435
|
+
# Fail-soft: any read/parse/resolution error degrades to the enabled
|
|
436
|
+
# default (the check renders OK regardless — it never FAILs/WARNs).
|
|
437
|
+
telemetry_enabled, telemetry_reason = (True, "enabled")
|
|
438
|
+
|
|
419
439
|
# config.json — RAW READ, never load_config(). load_config()
|
|
420
440
|
# auto-creates on first run AND silently falls back to defaults
|
|
421
441
|
# on corruption — both behaviors would hide diagnostic state
|
|
@@ -518,6 +538,10 @@ def doctor_gather_state(
|
|
|
518
538
|
is_dev_checkout=_cctally_core._is_dev_checkout(),
|
|
519
539
|
# Preview channel (CCTALLY_CHANNEL=preview): surfaced in install.mode.
|
|
520
540
|
channel=("preview" if _cctally_core.is_preview_channel() else "prod"),
|
|
541
|
+
# Anonymous install-count telemetry (spec 2026-07-07): read-only
|
|
542
|
+
# opt-out state, resolved above without minting an install_id.
|
|
543
|
+
telemetry_enabled=telemetry_enabled,
|
|
544
|
+
telemetry_reason=telemetry_reason,
|
|
521
545
|
# Pricing-freshness check (spec §5.1): trailing-30d coverage gaps.
|
|
522
546
|
pricing_coverage=pricing_coverage,
|
|
523
547
|
# Conversation-sessions rollup consistency (#217 S1 / U9).
|
package/bin/_cctally_parser.py
CHANGED
|
@@ -2225,6 +2225,47 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
2225
2225
|
cfg_unset.add_argument("key", help="Config key")
|
|
2226
2226
|
cfg_unset.set_defaults(func=c.cmd_config)
|
|
2227
2227
|
|
|
2228
|
+
# ---- telemetry (anonymous install-count opt-out, spec 2026-07-07) ----
|
|
2229
|
+
tele_p = sub.add_parser(
|
|
2230
|
+
"telemetry",
|
|
2231
|
+
help="Show or change anonymous install-count telemetry",
|
|
2232
|
+
formatter_class=CLIHelpFormatter,
|
|
2233
|
+
description=textwrap.dedent("""\
|
|
2234
|
+
Anonymous install-count telemetry: what it sends, and how to opt out.
|
|
2235
|
+
|
|
2236
|
+
cctally sends, at most once a day, a minimal beat: a one-way
|
|
2237
|
+
month-rotating token (never your install id), the client version,
|
|
2238
|
+
and a coarse OS family (macos/linux/windows/other). No IP, no
|
|
2239
|
+
username, no paths, no session content ever leaves the machine.
|
|
2240
|
+
|
|
2241
|
+
Actions:
|
|
2242
|
+
(none) Show the current state, resolved reason, what gets sent,
|
|
2243
|
+
and the token that would be used this month.
|
|
2244
|
+
on Enable telemetry (sets telemetry.enabled = true).
|
|
2245
|
+
off Disable telemetry (sets telemetry.enabled = false).
|
|
2246
|
+
reset Discard the local install id and mint a fresh one.
|
|
2247
|
+
|
|
2248
|
+
It is also disabled by CCTALLY_DISABLE_TELEMETRY=1, the DO_NOT_TRACK
|
|
2249
|
+
convention, and in dev checkouts.
|
|
2250
|
+
|
|
2251
|
+
Examples:
|
|
2252
|
+
cctally telemetry
|
|
2253
|
+
cctally telemetry --json
|
|
2254
|
+
cctally telemetry off
|
|
2255
|
+
cctally telemetry on
|
|
2256
|
+
cctally telemetry reset
|
|
2257
|
+
"""),
|
|
2258
|
+
)
|
|
2259
|
+
tele_p.add_argument(
|
|
2260
|
+
"action", nargs="?", choices=("on", "off", "reset"),
|
|
2261
|
+
help="on|off|reset (omit to show current status).",
|
|
2262
|
+
)
|
|
2263
|
+
tele_p.add_argument(
|
|
2264
|
+
"--json", dest="json", action="store_true",
|
|
2265
|
+
help="Emit the status as JSON (status output only).",
|
|
2266
|
+
)
|
|
2267
|
+
tele_p.set_defaults(func=c.cmd_telemetry)
|
|
2268
|
+
|
|
2228
2269
|
# ---- alerts (threshold-actions Task 6) ----
|
|
2229
2270
|
p_alerts = sub.add_parser(
|
|
2230
2271
|
"alerts",
|
|
@@ -2649,6 +2690,26 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
2649
2690
|
)
|
|
2650
2691
|
uc.set_defaults(func=c.cmd_update_check_internal)
|
|
2651
2692
|
|
|
2693
|
+
# ---- _telemetry-beat (internal — hidden, detached anonymous install-count worker) ----
|
|
2694
|
+
tb = sub.add_parser(
|
|
2695
|
+
"_telemetry-beat",
|
|
2696
|
+
help=argparse.SUPPRESS,
|
|
2697
|
+
formatter_class=CLIHelpFormatter,
|
|
2698
|
+
description=textwrap.dedent(
|
|
2699
|
+
"""\
|
|
2700
|
+
Internal subcommand: detached anonymous install-count beat
|
|
2701
|
+
worker, spawned broad-but-throttled from
|
|
2702
|
+
`_post_command_update_hooks` (spec 2026-07-07). A dedicated
|
|
2703
|
+
worker, decoupled from `_update-check` — it touches only the
|
|
2704
|
+
telemetry markers, never update-check state. Honours every
|
|
2705
|
+
opt-out (CCTALLY_DISABLE_TELEMETRY / DO_NOT_TRACK / config /
|
|
2706
|
+
dev checkout) via `resolve_telemetry_state`. Always returns 0;
|
|
2707
|
+
failures are swallowed.
|
|
2708
|
+
"""
|
|
2709
|
+
),
|
|
2710
|
+
)
|
|
2711
|
+
tb.set_defaults(func=c.cmd_telemetry_beat_internal)
|
|
2712
|
+
|
|
2652
2713
|
# ---- repair-symlinks (internal — hidden; npm-postinstall self-heal, issue #114) ----
|
|
2653
2714
|
rs = sub.add_parser(
|
|
2654
2715
|
"repair-symlinks",
|
package/bin/_cctally_setup.py
CHANGED
|
@@ -2138,7 +2138,25 @@ def _setup_install(args: argparse.Namespace) -> int:
|
|
|
2138
2138
|
out.append(" cctally tui # terminal dashboard")
|
|
2139
2139
|
out.append(" cctally setup --status # verify install state")
|
|
2140
2140
|
|
|
2141
|
+
# Install-time telemetry disclosure (spec 2026-07-07 §4). Text summary
|
|
2142
|
+
# only — the --json envelope carries a structured `telemetry` field
|
|
2143
|
+
# instead (never prose). Shown unconditionally as a factual disclosure of
|
|
2144
|
+
# the on-by-default, opt-out install-count beat; the opt-out command +
|
|
2145
|
+
# docs link are always surfaced so the fact is discoverable even when the
|
|
2146
|
+
# interactive first-run notice never fires (headless / statusline-only).
|
|
2147
|
+
out.append("")
|
|
2148
|
+
out.append("cctally counts anonymous active installs to gauge real usage.")
|
|
2149
|
+
out.append("What's sent: a rotating, un-linkable token + version + OS family.")
|
|
2150
|
+
out.append("No identity, no paths, no usage data, no IP stored. Auto-expires monthly.")
|
|
2151
|
+
out.append("Opt out anytime: cctally telemetry off (or CCTALLY_DISABLE_TELEMETRY=1)")
|
|
2152
|
+
out.append("How it works: https://github.com/omrikais/cctally/blob/main/docs/telemetry.md")
|
|
2153
|
+
|
|
2141
2154
|
if getattr(args, "json", False):
|
|
2155
|
+
# JSON-safe telemetry disclosure (spec 2026-07-07 §4): a structured
|
|
2156
|
+
# field, never prose. Resolved READ-ONLY via `resolve_telemetry_state`
|
|
2157
|
+
# (side-effect-free — mints no install_id, writes no config), so the
|
|
2158
|
+
# envelope reports the opt-out state without arming telemetry.
|
|
2159
|
+
tele_enabled, tele_reason = c.resolve_telemetry_state(c.load_config())
|
|
2142
2160
|
envelope = {
|
|
2143
2161
|
"schema_version": 1,
|
|
2144
2162
|
"mode": "install",
|
|
@@ -2174,6 +2192,10 @@ def _setup_install(args: argparse.Namespace) -> int:
|
|
|
2174
2192
|
"session_cache_rows": bootstrap_rows,
|
|
2175
2193
|
"oauth_status": bootstrap_oauth_status,
|
|
2176
2194
|
},
|
|
2195
|
+
"telemetry": {
|
|
2196
|
+
"enabled": tele_enabled,
|
|
2197
|
+
"reason": tele_reason,
|
|
2198
|
+
},
|
|
2177
2199
|
"warnings_count": warnings,
|
|
2178
2200
|
"exit_code": 0,
|
|
2179
2201
|
}
|