cctally 1.104.0 → 1.105.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +38 -0
- package/README.md +6 -6
- package/bin/_cctally_alerts.py +46 -13
- package/bin/_cctally_cache.py +193 -71
- package/bin/_cctally_dashboard.py +683 -70
- package/bin/_cctally_dashboard_conversation.py +400 -6
- package/bin/_cctally_dashboard_envelope.py +17 -0
- package/bin/_cctally_dashboard_perf.py +93 -0
- package/bin/_cctally_dashboard_sources.py +988 -94
- package/bin/_cctally_db.py +30 -3
- package/bin/_cctally_diagnosis_sources.py +699 -186
- package/bin/_cctally_doctor.py +22 -8
- package/bin/_cctally_forecast.py +17 -9
- package/bin/_cctally_journal.py +298 -40
- package/bin/_cctally_quota.py +15 -0
- package/bin/_cctally_quota_model.py +584 -58
- package/bin/_cctally_record.py +133 -67
- package/bin/_cctally_tui.py +317 -70
- package/bin/_lib_codex_conversation_query.py +258 -12
- package/bin/_lib_codex_hooks.py +26 -0
- package/bin/_lib_conversation_query.py +145 -15
- package/bin/_lib_dashboard_json.py +105 -0
- package/bin/_lib_diagnosis.py +21 -2
- package/bin/_lib_doctor.py +64 -8
- package/bin/_lib_ingest_frontier.py +889 -0
- package/bin/_lib_meter_rate_change.py +95 -29
- package/bin/_lib_perf.py +22 -0
- package/bin/_lib_pricing.py +30 -3
- package/bin/_lib_quota_copy.py +30 -4
- package/bin/_lib_quota_model.py +215 -28
- package/bin/_lib_rate_change_delivery.py +119 -0
- package/bin/_lib_record.py +50 -0
- package/bin/_lib_retained_size.py +187 -0
- package/bin/_lib_snapshot_cache.py +504 -17
- package/bin/_lib_statusline.py +3 -29
- package/bin/_lib_tick_stats.py +28 -4
- package/dashboard/static/assets/ConversationsView-DA4j7Yso.js +72 -0
- package/dashboard/static/assets/DoctorModal-D4PbVnbE.js +1 -0
- package/dashboard/static/assets/ModalRoot-D4oSPxPp.js +1 -0
- package/dashboard/static/assets/ProjectsDrillPanel-DkPlB9JH.js +1 -0
- package/dashboard/static/assets/SourceDetailModal-BE7FONsS.js +1 -0
- package/dashboard/static/assets/UpdateModal-BJuI8glf.js +7 -0
- package/dashboard/static/assets/index-w_bINkJ8.js +13 -0
- package/dashboard/static/assets/outlineNavigation-zDVm6Hdd.js +9 -0
- package/dashboard/static/assets/useKeymap-CJ-Pi17D.js +1 -0
- package/dashboard/static/dashboard.html +2 -1
- package/package.json +4 -1
- package/dashboard/static/assets/index-D19TO7Mg.js +0 -97
|
@@ -42,6 +42,13 @@ import re
|
|
|
42
42
|
import socket
|
|
43
43
|
import sqlite3
|
|
44
44
|
import sys
|
|
45
|
+
import base64
|
|
46
|
+
import collections
|
|
47
|
+
import dataclasses
|
|
48
|
+
import hashlib
|
|
49
|
+
import secrets
|
|
50
|
+
import threading
|
|
51
|
+
import time
|
|
45
52
|
|
|
46
53
|
from _cctally_cache import (
|
|
47
54
|
open_cache_db,
|
|
@@ -51,7 +58,11 @@ from _cctally_cache import (
|
|
|
51
58
|
sync_codex_conversations,
|
|
52
59
|
_codex_provider_roots,
|
|
53
60
|
)
|
|
54
|
-
from _lib_dashboard_json import
|
|
61
|
+
from _lib_dashboard_json import (
|
|
62
|
+
encode_dashboard_json,
|
|
63
|
+
encode_dashboard_json_bytes_capped,
|
|
64
|
+
)
|
|
65
|
+
from _lib_retained_size import retained_size_bytes
|
|
55
66
|
|
|
56
67
|
# Live-tail watch-loop tuning — used ONLY by _handle_get_conversation_events_impl
|
|
57
68
|
# below, so moved here with the events handler (spec §4.1 / §6).
|
|
@@ -61,6 +72,224 @@ _LIVE_TAIL_KEEPALIVE = 15.0 # idle keep-alive cadence (proxy guard)
|
|
|
61
72
|
_LIVE_TAIL_FILE_RESET_EVERY = 10 # re-resolve the session file set every N cycles
|
|
62
73
|
|
|
63
74
|
|
|
75
|
+
# #682 — progressive outline transport. The canonical outline remains the
|
|
76
|
+
# legacy endpoint body; an opt-in request gets a constant-size opaque transfer
|
|
77
|
+
# ticket. The first bounded chunk request materializes the canonical body, so
|
|
78
|
+
# the initial reader response never waits on or transfers the whole outline.
|
|
79
|
+
# Keeping the cache process-local avoids a second durable transcript surface.
|
|
80
|
+
_OUTLINE_TRANSFER_CHUNK = 192 * 1024
|
|
81
|
+
_OUTLINE_TRANSFER_ITEM_CAP = 32 * 1024 * 1024
|
|
82
|
+
_OUTLINE_TRANSFER_TOTAL_CAP = 64 * 1024 * 1024
|
|
83
|
+
_OUTLINE_TRANSFER_COUNT_CAP = 128
|
|
84
|
+
_OUTLINE_TRANSFER_TTL = 120.0
|
|
85
|
+
_OUTLINE_TRANSFERS = collections.OrderedDict()
|
|
86
|
+
_OUTLINE_TRANSFERS_BYTES = 0
|
|
87
|
+
_OUTLINE_TRANSFERS_EVICTIONS = 0
|
|
88
|
+
_OUTLINE_TRANSFERS_FALLBACKS = 0
|
|
89
|
+
_OUTLINE_TRANSFERS_LOCK = threading.Lock()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclasses.dataclass
|
|
93
|
+
class _OutlineTransferRecord:
|
|
94
|
+
created: float
|
|
95
|
+
builder: object
|
|
96
|
+
payload: bytes | None = None
|
|
97
|
+
digest: str | None = None
|
|
98
|
+
state: str = "pending"
|
|
99
|
+
cancelled: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
100
|
+
ready: threading.Event = dataclasses.field(default_factory=threading.Event)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _outline_transfer_estimated_bytes_locked() -> int:
|
|
104
|
+
return retained_size_bytes(
|
|
105
|
+
_OUTLINE_TRANSFERS, stop_after=_OUTLINE_TRANSFER_TOTAL_CAP)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def outline_transfer_cache_stats():
|
|
109
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
110
|
+
return {
|
|
111
|
+
"estimatedBytes": _outline_transfer_estimated_bytes_locked(),
|
|
112
|
+
"maxBytes": _OUTLINE_TRANSFER_TOTAL_CAP,
|
|
113
|
+
"payloadBytes": int(_OUTLINE_TRANSFERS_BYTES),
|
|
114
|
+
"entryCount": len(_OUTLINE_TRANSFERS),
|
|
115
|
+
"maxEntries": _OUTLINE_TRANSFER_COUNT_CAP,
|
|
116
|
+
"evictionCount": int(_OUTLINE_TRANSFERS_EVICTIONS),
|
|
117
|
+
"fallbackCount": int(_OUTLINE_TRANSFERS_FALLBACKS),
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _drop_outline_transfer_locked(token):
|
|
122
|
+
"""Remove one transfer and keep the materialized-byte count exact."""
|
|
123
|
+
global _OUTLINE_TRANSFERS_BYTES
|
|
124
|
+
record = _OUTLINE_TRANSFERS.pop(token, None)
|
|
125
|
+
if record is not None:
|
|
126
|
+
if record.payload is not None:
|
|
127
|
+
_OUTLINE_TRANSFERS_BYTES -= len(record.payload)
|
|
128
|
+
record.cancelled.set()
|
|
129
|
+
record.ready.set()
|
|
130
|
+
return record
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _prune_outline_transfers_locked(now):
|
|
134
|
+
expired = [token for token, record in _OUTLINE_TRANSFERS.items()
|
|
135
|
+
if now - record.created > _OUTLINE_TRANSFER_TTL]
|
|
136
|
+
for token in expired:
|
|
137
|
+
_drop_outline_transfer_locked(token)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _store_outline_transfer(builder):
|
|
141
|
+
"""Store a lazy canonical-body builder and return opaque metadata."""
|
|
142
|
+
global _OUTLINE_TRANSFERS_BYTES, _OUTLINE_TRANSFERS_EVICTIONS
|
|
143
|
+
now = time.monotonic()
|
|
144
|
+
token = secrets.token_urlsafe(24)
|
|
145
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
146
|
+
_prune_outline_transfers_locked(now)
|
|
147
|
+
# A deterministic token factory in tests, or a vanishingly unlikely
|
|
148
|
+
# production collision, cancels the prior generation before replacement.
|
|
149
|
+
_drop_outline_transfer_locked(token)
|
|
150
|
+
_OUTLINE_TRANSFERS[token] = _OutlineTransferRecord(now, builder)
|
|
151
|
+
while _OUTLINE_TRANSFERS and (
|
|
152
|
+
len(_OUTLINE_TRANSFERS) > _OUTLINE_TRANSFER_COUNT_CAP
|
|
153
|
+
or _outline_transfer_estimated_bytes_locked()
|
|
154
|
+
> _OUTLINE_TRANSFER_TOTAL_CAP
|
|
155
|
+
):
|
|
156
|
+
_drop_outline_transfer_locked(next(iter(_OUTLINE_TRANSFERS)))
|
|
157
|
+
_OUTLINE_TRANSFERS_EVICTIONS += 1
|
|
158
|
+
return {"token": token, "chunk_size": _OUTLINE_TRANSFER_CHUNK}
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _claim_outline_transfer(token):
|
|
162
|
+
"""Atomically claim a pending builder or join its in-flight generation."""
|
|
163
|
+
now = time.monotonic()
|
|
164
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
165
|
+
_prune_outline_transfers_locked(now)
|
|
166
|
+
record = _OUTLINE_TRANSFERS.get(token)
|
|
167
|
+
if record is None:
|
|
168
|
+
return None
|
|
169
|
+
_OUTLINE_TRANSFERS.move_to_end(token)
|
|
170
|
+
if record.payload is not None:
|
|
171
|
+
return "ready", record, None
|
|
172
|
+
if record.state == "pending":
|
|
173
|
+
record.state = "building"
|
|
174
|
+
return "build", record, record.builder
|
|
175
|
+
return "wait", record, None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _outline_transfer_builder(token):
|
|
179
|
+
"""Compatibility probe: claim one builder, ``False`` if joined/ready."""
|
|
180
|
+
claim = _claim_outline_transfer(token)
|
|
181
|
+
if claim is None:
|
|
182
|
+
return None
|
|
183
|
+
action, _record, builder = claim
|
|
184
|
+
return builder if action == "build" else False
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _materialize_outline_transfer(token, payload, *, claim=None):
|
|
188
|
+
"""Publish immutable canonical bytes into a still-live lazy transfer."""
|
|
189
|
+
global _OUTLINE_TRANSFERS_BYTES
|
|
190
|
+
global _OUTLINE_TRANSFERS_EVICTIONS, _OUTLINE_TRANSFERS_FALLBACKS
|
|
191
|
+
if len(payload) > _OUTLINE_TRANSFER_ITEM_CAP:
|
|
192
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
193
|
+
current = _OUTLINE_TRANSFERS.get(token)
|
|
194
|
+
if current is not None and (claim is None or current is claim):
|
|
195
|
+
_drop_outline_transfer_locked(token)
|
|
196
|
+
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
197
|
+
return "too_large"
|
|
198
|
+
now = time.monotonic()
|
|
199
|
+
digest = hashlib.sha256(payload).hexdigest()
|
|
200
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
201
|
+
_prune_outline_transfers_locked(now)
|
|
202
|
+
record = _OUTLINE_TRANSFERS.get(token)
|
|
203
|
+
if record is None or (claim is not None and record is not claim):
|
|
204
|
+
return "expired"
|
|
205
|
+
if record.cancelled.is_set():
|
|
206
|
+
return "expired"
|
|
207
|
+
if record.payload is not None:
|
|
208
|
+
return "ready"
|
|
209
|
+
while (_OUTLINE_TRANSFERS and
|
|
210
|
+
_OUTLINE_TRANSFERS_BYTES + len(payload) > _OUTLINE_TRANSFER_TOTAL_CAP):
|
|
211
|
+
victim = next(
|
|
212
|
+
((old_token, old_record)
|
|
213
|
+
for old_token, old_record in _OUTLINE_TRANSFERS.items()
|
|
214
|
+
if old_token != token),
|
|
215
|
+
None,
|
|
216
|
+
)
|
|
217
|
+
if victim is None:
|
|
218
|
+
break
|
|
219
|
+
old_token, _old_record = victim
|
|
220
|
+
_drop_outline_transfer_locked(old_token)
|
|
221
|
+
_OUTLINE_TRANSFERS_EVICTIONS += 1
|
|
222
|
+
record.payload = payload
|
|
223
|
+
record.digest = digest
|
|
224
|
+
record.builder = None
|
|
225
|
+
record.state = "ready"
|
|
226
|
+
_OUTLINE_TRANSFERS_BYTES += len(payload)
|
|
227
|
+
while (_OUTLINE_TRANSFERS
|
|
228
|
+
and _outline_transfer_estimated_bytes_locked()
|
|
229
|
+
> _OUTLINE_TRANSFER_TOTAL_CAP):
|
|
230
|
+
victim = next(
|
|
231
|
+
(old_token for old_token in _OUTLINE_TRANSFERS
|
|
232
|
+
if old_token != token),
|
|
233
|
+
None,
|
|
234
|
+
)
|
|
235
|
+
if victim is None:
|
|
236
|
+
_drop_outline_transfer_locked(token)
|
|
237
|
+
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
238
|
+
return "too_large"
|
|
239
|
+
_drop_outline_transfer_locked(victim)
|
|
240
|
+
_OUTLINE_TRANSFERS_EVICTIONS += 1
|
|
241
|
+
record.ready.set()
|
|
242
|
+
return "ready"
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _cancel_outline_transfer(token, *, claim=None) -> bool:
|
|
246
|
+
"""Cancel one exact transfer generation and wake every joined reader."""
|
|
247
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
248
|
+
current = _OUTLINE_TRANSFERS.get(token)
|
|
249
|
+
if current is None or (claim is not None and current is not claim):
|
|
250
|
+
return False
|
|
251
|
+
_drop_outline_transfer_locked(token)
|
|
252
|
+
return True
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _reject_outline_transfer_too_large(token, *, claim) -> bool:
|
|
256
|
+
"""Drop one exact over-cap generation without allocating a sentinel body."""
|
|
257
|
+
global _OUTLINE_TRANSFERS_FALLBACKS
|
|
258
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
259
|
+
current = _OUTLINE_TRANSFERS.get(token)
|
|
260
|
+
if current is None or current is not claim:
|
|
261
|
+
return False
|
|
262
|
+
_drop_outline_transfer_locked(token)
|
|
263
|
+
_OUTLINE_TRANSFERS_FALLBACKS += 1
|
|
264
|
+
return True
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _read_outline_transfer(token, offset):
|
|
268
|
+
"""Return one immutable chunk, or ``None`` when the token is gone."""
|
|
269
|
+
now = time.monotonic()
|
|
270
|
+
with _OUTLINE_TRANSFERS_LOCK:
|
|
271
|
+
_prune_outline_transfers_locked(now)
|
|
272
|
+
record = _OUTLINE_TRANSFERS.get(token)
|
|
273
|
+
if record is None:
|
|
274
|
+
return None
|
|
275
|
+
payload, digest = record.payload, record.digest
|
|
276
|
+
_OUTLINE_TRANSFERS.move_to_end(token)
|
|
277
|
+
if payload is None:
|
|
278
|
+
return None
|
|
279
|
+
if offset < 0 or offset > len(payload):
|
|
280
|
+
raise ValueError("offset out of range")
|
|
281
|
+
chunk = payload[offset:offset + _OUTLINE_TRANSFER_CHUNK]
|
|
282
|
+
next_offset = offset + len(chunk)
|
|
283
|
+
return {
|
|
284
|
+
"offset": offset,
|
|
285
|
+
"next_offset": next_offset,
|
|
286
|
+
"total": len(payload),
|
|
287
|
+
"sha256": digest,
|
|
288
|
+
"done": next_offset == len(payload),
|
|
289
|
+
"chunk": base64.b64encode(chunk).decode("ascii"),
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
|
|
64
293
|
# Module-local forwarders for the two generic query-string helpers that STAY
|
|
65
294
|
# in the dashboard (24 call sites in the moved handlers; the dashboard is fully
|
|
66
295
|
# loaded before any handler runs, so the call-time reach is cycle-free).
|
|
@@ -450,7 +679,9 @@ def _parse_search_kind_impl(handler, q, valid=_CONV_SEARCH_KINDS):
|
|
|
450
679
|
return None
|
|
451
680
|
return kind
|
|
452
681
|
|
|
453
|
-
def _run_conversation_query_impl(
|
|
682
|
+
def _run_conversation_query_impl(
|
|
683
|
+
handler, kernel_call, log_label, *, cancelled=None,
|
|
684
|
+
):
|
|
454
685
|
"""Open conversations.db, run ``kernel_call(conn)``, close.
|
|
455
686
|
|
|
456
687
|
Collapses the triplicated open-store → try/except/finally → 500
|
|
@@ -487,12 +718,25 @@ def _run_conversation_query_impl(handler, kernel_call, log_label):
|
|
|
487
718
|
handler._respond_json(400, {"error": "invalid account"})
|
|
488
719
|
return False, None
|
|
489
720
|
scope_conversations_db_to_account(conn, account_vals[0])
|
|
721
|
+
if cancelled is not None:
|
|
722
|
+
conn.set_progress_handler(lambda: 1 if cancelled() else 0, 1000)
|
|
723
|
+
if cancelled():
|
|
724
|
+
return None, None
|
|
490
725
|
body = kernel_call(conn)
|
|
726
|
+
if cancelled is not None and cancelled():
|
|
727
|
+
return None, None
|
|
491
728
|
except Exception as exc: # noqa: BLE001
|
|
729
|
+
if cancelled is not None and cancelled():
|
|
730
|
+
return None, None
|
|
492
731
|
handler.log_error("%s failed: %r", log_label, exc)
|
|
493
732
|
handler._respond_json(500, {"error": f"{type(exc).__name__}: {exc}"})
|
|
494
733
|
return False, None
|
|
495
734
|
finally:
|
|
735
|
+
if cancelled is not None:
|
|
736
|
+
try:
|
|
737
|
+
conn.set_progress_handler(None, 0)
|
|
738
|
+
except sqlite3.Error:
|
|
739
|
+
pass
|
|
496
740
|
conn.close()
|
|
497
741
|
return True, body
|
|
498
742
|
|
|
@@ -1261,6 +1505,12 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
|
1261
1505
|
if not handler._require_transcripts_allowed():
|
|
1262
1506
|
return
|
|
1263
1507
|
import urllib.parse as _u
|
|
1508
|
+
query = _u.parse_qs(_u.urlsplit(handler.path).query, keep_blank_values=True)
|
|
1509
|
+
progressive_values = query.get("progressive")
|
|
1510
|
+
if progressive_values is not None and progressive_values != ["1"]:
|
|
1511
|
+
handler._respond_json(400, {"error": "invalid progressive param"})
|
|
1512
|
+
return
|
|
1513
|
+
progressive = progressive_values == ["1"]
|
|
1264
1514
|
session_id = _u.unquote(path[len("/api/conversation/"):-len("/outline")])
|
|
1265
1515
|
if not session_id:
|
|
1266
1516
|
handler.send_error(404, "conversation not found")
|
|
@@ -1268,11 +1518,75 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
|
1268
1518
|
if session_id.startswith("v1."):
|
|
1269
1519
|
speed = _resolve_effective_speed()
|
|
1270
1520
|
disp = _conversation_dispatch()
|
|
1271
|
-
|
|
1272
|
-
handler
|
|
1273
|
-
|
|
1274
|
-
|
|
1521
|
+
if not progressive:
|
|
1522
|
+
ok, body = handler._run_conversation_query(
|
|
1523
|
+
lambda conn: disp.neutral_outline(
|
|
1524
|
+
conn, session_id, effective_speed=speed),
|
|
1525
|
+
"/api/conversation/outline")
|
|
1526
|
+
if not ok:
|
|
1527
|
+
return
|
|
1528
|
+
_respond_qualified_json(handler, body)
|
|
1529
|
+
return
|
|
1530
|
+
ok, preflight = handler._run_conversation_query(
|
|
1531
|
+
lambda conn: disp.neutral_events_preflight(conn, session_id),
|
|
1275
1532
|
"/api/conversation/outline")
|
|
1533
|
+
if not ok:
|
|
1534
|
+
return
|
|
1535
|
+
if (preflight or {}).get("status") != "ok":
|
|
1536
|
+
_respond_qualified_json(handler, preflight)
|
|
1537
|
+
return
|
|
1538
|
+
account_values = query.get("account")
|
|
1539
|
+
account_key = account_values[0] if account_values else None
|
|
1540
|
+
|
|
1541
|
+
def _build_qualified(transfer_handler, cancelled):
|
|
1542
|
+
def _query(conn):
|
|
1543
|
+
if account_key is not None:
|
|
1544
|
+
scope_conversations_db_to_account(conn, account_key)
|
|
1545
|
+
return disp.neutral_outline(
|
|
1546
|
+
conn, session_id, effective_speed=speed)
|
|
1547
|
+
return _run_conversation_query_impl(
|
|
1548
|
+
transfer_handler,
|
|
1549
|
+
_query,
|
|
1550
|
+
"/api/conversation/outline-transfer",
|
|
1551
|
+
cancelled=cancelled,
|
|
1552
|
+
)
|
|
1553
|
+
|
|
1554
|
+
handler._respond_json(200, {
|
|
1555
|
+
"progressive": 1,
|
|
1556
|
+
"transfer": _store_outline_transfer(_build_qualified),
|
|
1557
|
+
})
|
|
1558
|
+
return
|
|
1559
|
+
if progressive:
|
|
1560
|
+
ok, exists = handler._run_conversation_query(
|
|
1561
|
+
lambda conn: conn.execute(
|
|
1562
|
+
"SELECT 1 FROM conversation_messages WHERE session_id=? LIMIT 1",
|
|
1563
|
+
(session_id,),
|
|
1564
|
+
).fetchone() is not None,
|
|
1565
|
+
"/api/conversation/outline")
|
|
1566
|
+
if not ok:
|
|
1567
|
+
return
|
|
1568
|
+
if not exists:
|
|
1569
|
+
handler.send_error(404, "conversation not found")
|
|
1570
|
+
return
|
|
1571
|
+
|
|
1572
|
+
def _build_bare(transfer_handler, cancelled):
|
|
1573
|
+
def _query(conn):
|
|
1574
|
+
account_values = query.get("account")
|
|
1575
|
+
if account_values:
|
|
1576
|
+
scope_conversations_db_to_account(conn, account_values[0])
|
|
1577
|
+
return transfer_handler._conversation_query(
|
|
1578
|
+
).get_conversation_outline(conn, session_id)
|
|
1579
|
+
return _run_conversation_query_impl(
|
|
1580
|
+
transfer_handler,
|
|
1581
|
+
_query,
|
|
1582
|
+
"/api/conversation/outline-transfer",
|
|
1583
|
+
cancelled=cancelled,
|
|
1584
|
+
)
|
|
1585
|
+
|
|
1586
|
+
handler._respond_json(200, {
|
|
1587
|
+
"progressive": 1,
|
|
1588
|
+
"transfer": _store_outline_transfer(_build_bare),
|
|
1589
|
+
})
|
|
1276
1590
|
return
|
|
1277
1591
|
ok, body = handler._run_conversation_query(
|
|
1278
1592
|
lambda conn: handler._conversation_query().get_conversation_outline(conn, session_id),
|
|
@@ -1284,6 +1598,86 @@ def _handle_get_conversation_outline_impl(handler, path: str) -> None:
|
|
|
1284
1598
|
return
|
|
1285
1599
|
handler._respond_json(200, body)
|
|
1286
1600
|
|
|
1601
|
+
|
|
1602
|
+
def _handle_get_conversation_outline_transfer_impl(handler, path: str) -> None:
|
|
1603
|
+
"""Serve one bounded chunk from an opaque, immutable outline transfer."""
|
|
1604
|
+
if not handler._require_transcripts_allowed():
|
|
1605
|
+
return
|
|
1606
|
+
import urllib.parse as _u
|
|
1607
|
+
token = _u.unquote(path[len("/api/conversation/outline-transfer/"):])
|
|
1608
|
+
if not token or not re.fullmatch(r"[A-Za-z0-9_-]+", token):
|
|
1609
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1610
|
+
return
|
|
1611
|
+
parsed = _u.parse_qs(_u.urlsplit(handler.path).query, keep_blank_values=True)
|
|
1612
|
+
values = parsed.get("offset")
|
|
1613
|
+
if values is None:
|
|
1614
|
+
offset = 0
|
|
1615
|
+
elif len(values) != 1 or not values[0].isascii() or not values[0].isdigit():
|
|
1616
|
+
handler._respond_json(400, {"error": "invalid offset"})
|
|
1617
|
+
return
|
|
1618
|
+
else:
|
|
1619
|
+
offset = int(values[0])
|
|
1620
|
+
claim = _claim_outline_transfer(token)
|
|
1621
|
+
if claim is None:
|
|
1622
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1623
|
+
return
|
|
1624
|
+
action, record, builder = claim
|
|
1625
|
+
if action == "build":
|
|
1626
|
+
ok, full_body = builder(handler, record.cancelled.is_set)
|
|
1627
|
+
if ok is None:
|
|
1628
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1629
|
+
return
|
|
1630
|
+
if not ok:
|
|
1631
|
+
_cancel_outline_transfer(token, claim=record)
|
|
1632
|
+
return
|
|
1633
|
+
if full_body is None:
|
|
1634
|
+
_cancel_outline_transfer(token, claim=record)
|
|
1635
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1636
|
+
return
|
|
1637
|
+
payload = encode_dashboard_json_bytes_capped(
|
|
1638
|
+
full_body, max_bytes=_OUTLINE_TRANSFER_ITEM_CAP)
|
|
1639
|
+
if payload is None:
|
|
1640
|
+
_reject_outline_transfer_too_large(token, claim=record)
|
|
1641
|
+
state = "too_large"
|
|
1642
|
+
else:
|
|
1643
|
+
state = _materialize_outline_transfer(token, payload, claim=record)
|
|
1644
|
+
if state == "too_large":
|
|
1645
|
+
handler._respond_json(
|
|
1646
|
+
413, {"error": "outline exceeds progressive transfer limit"})
|
|
1647
|
+
return
|
|
1648
|
+
if state == "expired":
|
|
1649
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1650
|
+
return
|
|
1651
|
+
elif action == "wait":
|
|
1652
|
+
record.ready.wait(_OUTLINE_TRANSFER_TTL)
|
|
1653
|
+
if record.cancelled.is_set() or record.payload is None:
|
|
1654
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1655
|
+
return
|
|
1656
|
+
try:
|
|
1657
|
+
body = _read_outline_transfer(token, offset)
|
|
1658
|
+
except ValueError:
|
|
1659
|
+
handler._respond_json(400, {"error": "offset out of range"})
|
|
1660
|
+
return
|
|
1661
|
+
if body is None:
|
|
1662
|
+
handler._respond_json(410, {"error": "outline transfer expired"})
|
|
1663
|
+
return
|
|
1664
|
+
handler._respond_json(200, body)
|
|
1665
|
+
|
|
1666
|
+
|
|
1667
|
+
def _handle_delete_conversation_outline_transfer_impl(handler, path: str) -> None:
|
|
1668
|
+
"""Best-effort cancellation for a browser-abandoned progressive token."""
|
|
1669
|
+
if not handler._require_transcripts_allowed():
|
|
1670
|
+
return
|
|
1671
|
+
import urllib.parse as _u
|
|
1672
|
+
token = _u.unquote(path[len("/api/conversation/outline-transfer/"):])
|
|
1673
|
+
if not token or not re.fullmatch(r"[A-Za-z0-9_-]+", token):
|
|
1674
|
+
handler._respond_json(400, {"error": "invalid outline transfer token"})
|
|
1675
|
+
return
|
|
1676
|
+
_cancel_outline_transfer(token)
|
|
1677
|
+
handler.send_response(204)
|
|
1678
|
+
handler.send_header("Content-Length", "0")
|
|
1679
|
+
handler.end_headers()
|
|
1680
|
+
|
|
1287
1681
|
def _handle_get_conversation_prompts_impl(handler, path: str) -> None:
|
|
1288
1682
|
"""``GET /api/conversation/<sid>/prompts`` — ordered main-thread human
|
|
1289
1683
|
prompts + full text (#217 S7 F10, the session-comparison spine). Same
|
|
@@ -1213,6 +1213,22 @@ def _cause_presentation(code):
|
|
|
1213
1213
|
return out or None
|
|
1214
1214
|
|
|
1215
1215
|
|
|
1216
|
+
def _basis_presentation(basis):
|
|
1217
|
+
"""The shared two-register projection-basis copy, or None.
|
|
1218
|
+
|
|
1219
|
+
Separate from `_cause_presentation`: a basis describes a present figure's
|
|
1220
|
+
provenance and never joins the withholding-cause vocabulary.
|
|
1221
|
+
"""
|
|
1222
|
+
if not basis:
|
|
1223
|
+
return None
|
|
1224
|
+
try:
|
|
1225
|
+
copy = sys.modules["cctally"]._load_sibling("_lib_quota_copy")
|
|
1226
|
+
except Exception: # noqa: BLE001
|
|
1227
|
+
return None
|
|
1228
|
+
out = copy.basis_presentation(basis)
|
|
1229
|
+
return out or None
|
|
1230
|
+
|
|
1231
|
+
|
|
1216
1232
|
def _forecast_quota_envelope(fc, cw, rate_change):
|
|
1217
1233
|
"""Spec §10's optional typed `forecast.quota` object.
|
|
1218
1234
|
|
|
@@ -1276,6 +1292,7 @@ def _forecast_quota_envelope(fc, cw, rate_change):
|
|
|
1276
1292
|
calibration_code = calibration_code or "unavailable"
|
|
1277
1293
|
return {
|
|
1278
1294
|
"basis": basis,
|
|
1295
|
+
"basis_presentation": _basis_presentation(basis),
|
|
1279
1296
|
"projection_pct": projection,
|
|
1280
1297
|
"right_censored": bool(getattr(fc, "right_censored", False)),
|
|
1281
1298
|
# The cause of a WITHHELD projection, and separately why the
|
|
@@ -47,6 +47,7 @@ _TRACE_PATH = "/api/debug/backend/trace"
|
|
|
47
47
|
_REPORTED_REGIMES = ("active", "idle")
|
|
48
48
|
|
|
49
49
|
_REGIME_LABELS = {"active": "Codex-active", "idle": "Codex-idle"}
|
|
50
|
+
COMBINED_BACKGROUND_CPU_CEILING = 0.75
|
|
50
51
|
|
|
51
52
|
|
|
52
53
|
def _cctally():
|
|
@@ -202,6 +203,36 @@ def _mean_or_none(values):
|
|
|
202
203
|
return int(sum(values) / len(values)) if values else None
|
|
203
204
|
|
|
204
205
|
|
|
206
|
+
def _cpu_duty(rows):
|
|
207
|
+
paired = [
|
|
208
|
+
(int(row.get("cpu_ns") or 0), int(row["period_ns"]))
|
|
209
|
+
for row in rows or ()
|
|
210
|
+
if row.get("cpu_ns") is not None
|
|
211
|
+
and row.get("period_ns") is not None
|
|
212
|
+
and int(row["period_ns"]) > 0
|
|
213
|
+
]
|
|
214
|
+
if not paired:
|
|
215
|
+
return None
|
|
216
|
+
return sum(cpu for cpu, _period in paired) / sum(
|
|
217
|
+
period for _cpu, period in paired)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def summarise_combined_background_duty(tick: dict) -> dict:
|
|
221
|
+
"""Conservative sum of each loop's same-span one-core duty."""
|
|
222
|
+
main = _cpu_duty(tick.get("records") or ())
|
|
223
|
+
conversations = _cpu_duty(tick.get("conversation_sync") or ())
|
|
224
|
+
combined = None if main is None or conversations is None else (
|
|
225
|
+
main + conversations)
|
|
226
|
+
return {
|
|
227
|
+
"main": main,
|
|
228
|
+
"conversation": conversations,
|
|
229
|
+
"combined": combined,
|
|
230
|
+
"ceiling": COMBINED_BACKGROUND_CPU_CEILING,
|
|
231
|
+
"withinCeiling": None if combined is None else (
|
|
232
|
+
combined <= COMBINED_BACKGROUND_CPU_CEILING),
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
|
|
205
236
|
def _median(values):
|
|
206
237
|
import statistics
|
|
207
238
|
|
|
@@ -265,6 +296,17 @@ def _render_conversation_sync(tick: dict) -> list:
|
|
|
265
296
|
statuses[key] = statuses.get(key, 0) + 1
|
|
266
297
|
detail = " · ".join(f"{k} {v}" for k, v in sorted(statuses.items()))
|
|
267
298
|
lines.append(f" {'status':<14} {detail}")
|
|
299
|
+
for provider in ("claude", "codex"):
|
|
300
|
+
modes: dict = {}
|
|
301
|
+
files = 0
|
|
302
|
+
for record in rows:
|
|
303
|
+
mode = record.get(f"{provider}_mode") or "not_observed"
|
|
304
|
+
modes[mode] = modes.get(mode, 0) + 1
|
|
305
|
+
files += max(0, int(record.get(f"{provider}_files") or 0))
|
|
306
|
+
mode_detail = " · ".join(
|
|
307
|
+
f"{mode} {count}" for mode, count in sorted(modes.items()))
|
|
308
|
+
lines.append(
|
|
309
|
+
f" {provider:<14} {mode_detail} · files {files}")
|
|
268
310
|
return lines
|
|
269
311
|
|
|
270
312
|
|
|
@@ -316,6 +358,23 @@ def render_dashboard_perf(payload: dict) -> str:
|
|
|
316
358
|
f"over {len(builder)} tick(s)")
|
|
317
359
|
lines.append(
|
|
318
360
|
f" whole tick mean {_millis(_mean_or_none(duration))}")
|
|
361
|
+
cpu = [int(r.get("cpu_ns") or 0) for r in records]
|
|
362
|
+
lines.append(
|
|
363
|
+
f" thread cpu mean {_millis(_mean_or_none(cpu))}")
|
|
364
|
+
paired_cpu = [
|
|
365
|
+
(int(r.get("cpu_ns") or 0), int(r["period_ns"]))
|
|
366
|
+
for r in records
|
|
367
|
+
if r.get("cpu_ns") is not None
|
|
368
|
+
and r.get("period_ns") is not None
|
|
369
|
+
and int(r["period_ns"]) > 0
|
|
370
|
+
]
|
|
371
|
+
if paired_cpu:
|
|
372
|
+
duty = sum(value for value, _period in paired_cpu) / sum(
|
|
373
|
+
period for _value, period in paired_cpu
|
|
374
|
+
)
|
|
375
|
+
lines.append(f" cpu duty {duty * 100:.1f}% of one core")
|
|
376
|
+
else:
|
|
377
|
+
lines.append(" cpu duty no samples yet")
|
|
319
378
|
# #583 S5 §2.4: the cache.db read pin, measured at its own BEGIN and
|
|
320
379
|
# ROLLBACK boundaries. Reported separately from `builder` because it
|
|
321
380
|
# is a SUBSET of builder time rather than a third exclusive half, and
|
|
@@ -331,6 +390,7 @@ def render_dashboard_perf(payload: dict) -> str:
|
|
|
331
390
|
f"{'/cold' if newest.get('cold') else '/warm'} "
|
|
332
391
|
f"ingest {_millis(newest.get('ingest_ns'))} "
|
|
333
392
|
f"builder {_millis(newest.get('builder_ns'))} "
|
|
393
|
+
f"cpu {_millis(newest.get('cpu_ns'))} "
|
|
334
394
|
f"pin {_millis(newest.get('cache_pin_ns', 0) or 0)} "
|
|
335
395
|
f"total {_millis(newest.get('duration_ns'))}")
|
|
336
396
|
else:
|
|
@@ -342,6 +402,15 @@ def render_dashboard_perf(payload: dict) -> str:
|
|
|
342
402
|
f"total {_millis(standalone.get('duration_ns'))} "
|
|
343
403
|
f"(the last build made outside a refresh tick)")
|
|
344
404
|
lines.extend(_render_conversation_sync(tick))
|
|
405
|
+
combined = summarise_combined_background_duty(tick)
|
|
406
|
+
lines.extend(["", "Combined background work"])
|
|
407
|
+
if combined["combined"] is None:
|
|
408
|
+
lines.append(" duty not jointly sampled")
|
|
409
|
+
else:
|
|
410
|
+
verdict = "within" if combined["withinCeiling"] else "over"
|
|
411
|
+
lines.append(
|
|
412
|
+
f" duty {combined['combined'] * 100:.1f}% of one core · "
|
|
413
|
+
f"ceiling {combined['ceiling'] * 100:.1f}% · {verdict}")
|
|
345
414
|
lines.append("")
|
|
346
415
|
|
|
347
416
|
counts = tick.get("dispatch_counts") or {}
|
|
@@ -359,6 +428,24 @@ def render_dashboard_perf(payload: dict) -> str:
|
|
|
359
428
|
else:
|
|
360
429
|
lines.append("Group A cache-open failures none")
|
|
361
430
|
|
|
431
|
+
memory = payload.get("memory") or {}
|
|
432
|
+
if memory:
|
|
433
|
+
lines.append("")
|
|
434
|
+
lines.append("Retained process memory")
|
|
435
|
+
lines.append(
|
|
436
|
+
" owned "
|
|
437
|
+
f"{int(memory.get('ownerEstimatedBytes', 0)) / 1048576:.1f} MiB "
|
|
438
|
+
f"of {int(memory.get('ownerCeilingBytes', 0)) / 1048576:.1f} MiB")
|
|
439
|
+
lines.append(
|
|
440
|
+
" process cap "
|
|
441
|
+
f"{int(memory.get('processCeilingBytes', 0)) / 1048576:.0f} MiB")
|
|
442
|
+
for name, row in sorted((memory.get("owners") or {}).items()):
|
|
443
|
+
lines.append(
|
|
444
|
+
f" {name:<22} "
|
|
445
|
+
f"{int(row.get('estimatedBytes', 0)) / 1048576:.1f}/"
|
|
446
|
+
f"{int(row.get('maxBytes', 0)) / 1048576:.1f} MiB · "
|
|
447
|
+
f"entries {int(row.get('entryCount', 0))}")
|
|
448
|
+
|
|
362
449
|
applied = tracing.get("applied")
|
|
363
450
|
requested = tracing.get("requested")
|
|
364
451
|
applies_at = tracing.get("applies_at", "none")
|
|
@@ -374,6 +461,12 @@ def render_dashboard_perf(payload: dict) -> str:
|
|
|
374
461
|
else:
|
|
375
462
|
lines.append(" no phase tree stored — arm one with "
|
|
376
463
|
"`cctally dashboard-perf --trace on`")
|
|
464
|
+
if payload.get("ingest_phases") is not None:
|
|
465
|
+
ingest = payload["ingest_phases"]
|
|
466
|
+
lines.append(
|
|
467
|
+
f" ingest tree {ingest.get('elapsed_ms', 0):.1f}ms, generated at "
|
|
468
|
+
f"{payload.get('ingest_generated_at')}"
|
|
469
|
+
)
|
|
377
470
|
return "\n".join(lines) + "\n"
|
|
378
471
|
|
|
379
472
|
|