cctally 1.88.2 → 1.89.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/bin/_cctally_cache.py +827 -37
- package/bin/_cctally_config.py +125 -0
- package/bin/_cctally_core.py +86 -2
- package/bin/_cctally_dashboard_cache_report.py +31 -0
- package/bin/_cctally_dashboard_conversation.py +26 -7
- package/bin/_cctally_dashboard_sources.py +51 -0
- package/bin/_cctally_db.py +626 -0
- package/bin/_cctally_doctor.py +84 -1
- package/bin/_cctally_journal.py +2 -1
- package/bin/_cctally_parser.py +42 -0
- package/bin/_cctally_quota.py +1358 -112
- package/bin/_cctally_record.py +249 -9
- package/bin/_cctally_setup.py +14 -5
- package/bin/_cctally_store.py +16 -1
- package/bin/_cctally_tui.py +16 -2
- package/bin/_cctally_update.py +9 -2
- package/bin/_lib_background_mcp.py +168 -0
- package/bin/_lib_cache_report.py +19 -2
- package/bin/_lib_codex_conversation.py +8 -0
- package/bin/_lib_codex_conversation_query.py +8 -7
- package/bin/_lib_conversation.py +105 -5
- package/bin/_lib_conversation_dispatch.py +15 -4
- package/bin/_lib_conversation_query.py +294 -2
- package/bin/_lib_dashboard_sources.py +5 -1
- package/bin/_lib_doctor.py +202 -1
- package/bin/_lib_jsonl.py +12 -0
- package/bin/_lib_quota_alert_axes.py +188 -0
- package/bin/_lib_quota_ledger.py +274 -0
- package/bin/_lib_snapshot_cache.py +36 -0
- package/bin/cctally +6 -3
- package/dashboard/static/assets/index-BgoYXdus.js +92 -0
- package/dashboard/static/assets/index-Ub8vwz1M.css +1 -0
- package/dashboard/static/dashboard.html +2 -2
- package/package.json +4 -1
- package/dashboard/static/assets/index-B0ZCsoxI.css +0 -1
- package/dashboard/static/assets/index-Bvp8mxtz.js +0 -92
package/bin/_lib_cache_report.py
CHANGED
|
@@ -119,6 +119,11 @@ CACHE_REPORT_MIN_BASELINE_SESSIONS = 10
|
|
|
119
119
|
# dashboard/web/src/types/envelope.ts:71 — keeps the two surfaces in
|
|
120
120
|
# lockstep so a typo on either side fails type-check.
|
|
121
121
|
CacheAnomalyReason = Literal["net_negative", "cache_drop"]
|
|
122
|
+
# Every predicate _classify_anomalies can run, in reason-append order. The
|
|
123
|
+
# TypeScript mirror is CACHE_ANOMALY_PREDICATES in cacheReportVerdict.ts.
|
|
124
|
+
CACHE_ANOMALY_PREDICATES: tuple[CacheAnomalyReason, ...] = (
|
|
125
|
+
"net_negative", "cache_drop",
|
|
126
|
+
)
|
|
122
127
|
|
|
123
128
|
|
|
124
129
|
@dataclass
|
|
@@ -185,6 +190,10 @@ class CacheRow:
|
|
|
185
190
|
# Anomaly (populated by _classify_anomalies)
|
|
186
191
|
anomaly_triggered: bool = False
|
|
187
192
|
anomaly_reasons: list[CacheAnomalyReason] = field(default_factory=list)
|
|
193
|
+
# Predicates the classifier did NOT run for this row. Empty means every
|
|
194
|
+
# predicate was evaluated; a non-empty list is what lets the dashboard
|
|
195
|
+
# distinguish "evaluated, clean" from "never evaluated" (issue #443 F2).
|
|
196
|
+
anomaly_unevaluated: list[CacheAnomalyReason] = field(default_factory=list)
|
|
188
197
|
|
|
189
198
|
@property
|
|
190
199
|
def total_tokens(self) -> int:
|
|
@@ -767,6 +776,7 @@ def _classify_anomalies(
|
|
|
767
776
|
for row in rows:
|
|
768
777
|
row.anomaly_triggered = False
|
|
769
778
|
row.anomaly_reasons = []
|
|
779
|
+
row.anomaly_unevaluated = list(CACHE_ANOMALY_PREDICATES)
|
|
770
780
|
return
|
|
771
781
|
if not rows:
|
|
772
782
|
return
|
|
@@ -782,24 +792,31 @@ def _classify_anomalies(
|
|
|
782
792
|
|
|
783
793
|
for i, row in enumerate(rows):
|
|
784
794
|
reasons: list[CacheAnomalyReason] = []
|
|
795
|
+
unevaluated: list[CacheAnomalyReason] = []
|
|
785
796
|
|
|
786
797
|
# Trigger 1: net_negative (no baseline needed; cache-activity guard).
|
|
787
798
|
if row.cache_creation_tokens + row.cache_read_tokens > 0:
|
|
788
799
|
if row.net_usd < 0:
|
|
789
800
|
reasons.append("net_negative")
|
|
801
|
+
else:
|
|
802
|
+
unevaluated.append("net_negative")
|
|
790
803
|
|
|
791
804
|
# Trigger 2: cache_drop (requires baseline).
|
|
792
805
|
anchor = anchors[i]
|
|
806
|
+
median = None
|
|
793
807
|
if anchor is not None:
|
|
794
808
|
median = _compute_baseline_median(
|
|
795
809
|
rows, anchor=anchor,
|
|
796
810
|
window_days=window_days, min_samples=min_baseline,
|
|
797
811
|
exclude_row=row, is_session_mode=is_session_mode,
|
|
798
812
|
)
|
|
799
|
-
|
|
800
|
-
|
|
813
|
+
if median is None:
|
|
814
|
+
unevaluated.append("cache_drop")
|
|
815
|
+
elif (median - row.cache_hit_percent) >= threshold_pp:
|
|
816
|
+
reasons.append("cache_drop")
|
|
801
817
|
|
|
802
818
|
row.anomaly_reasons = reasons
|
|
819
|
+
row.anomaly_unevaluated = unevaluated
|
|
803
820
|
row.anomaly_triggered = bool(reasons)
|
|
804
821
|
|
|
805
822
|
|
|
@@ -90,6 +90,14 @@ CODEX_CONVERSATION_REPLAY_FROM_ZERO_KEY = (
|
|
|
90
90
|
# `CODEX_REPLAY_FROM_ZERO_KEY` — the replay is stalled, not merely pending, and
|
|
91
91
|
# Codex transcript ingest is deferred behind it. Read by `doctor`.
|
|
92
92
|
CODEX_REPLAY_BLOCKED_KEY = "codex_replay_from_zero_blocked"
|
|
93
|
+
# Written when a BUDGETED tick declined the replay outright (public #5 §4). A
|
|
94
|
+
# byte-zero replay is not sliceable, so the hook path never attempts one — but
|
|
95
|
+
# that decline returns before the ingest-backlog record is written, so without
|
|
96
|
+
# this key a hook-only install whose every Codex sync is frozen reads as a
|
|
97
|
+
# drained store on `doctor`, in the lifecycle line and in the dashboard
|
|
98
|
+
# envelope. `{since, at}`: `since` is when the freeze started and carries
|
|
99
|
+
# forward across ticks, `at` is the most recent decline.
|
|
100
|
+
CODEX_REPLAY_DEFERRED_KEY = "codex_replay_from_zero_deferred"
|
|
93
101
|
|
|
94
102
|
# Structural wrapper prefixes skipped during title selection (§4.3), pinned from
|
|
95
103
|
# the corpus (title-wrapper-window). Prefix-structural, never content heuristics.
|
|
@@ -1005,7 +1005,11 @@ def codex_conversation_source_paths(
|
|
|
1005
1005
|
# ── detail assembly (§5.2 / §5.4 / §5.6) ──────────────────────────────────────
|
|
1006
1006
|
|
|
1007
1007
|
|
|
1008
|
-
def _paginate_items(items: list[dict], *, after, before, tail, limit):
|
|
1008
|
+
def _paginate_items(items: list[dict], *, after, before, tail: bool, limit: int):
|
|
1009
|
+
# ``tail`` is the flag the HTTP layer parses out of ``?tail=1`` — which page
|
|
1010
|
+
# of ``limit`` to cut, never how many items to return. Treating it as a count
|
|
1011
|
+
# made ``min(True, limit)`` serve a one-item tail page, and ``tail=False``
|
|
1012
|
+
# (the shape every non-tail request arrives in) skip ``limit`` altogether.
|
|
1009
1013
|
keys = [it["item_key"] for it in items]
|
|
1010
1014
|
aliases = {
|
|
1011
1015
|
alias: index
|
|
@@ -1022,11 +1026,8 @@ def _paginate_items(items: list[dict], *, after, before, tail, limit):
|
|
|
1022
1026
|
elif before is not None and before in aliases:
|
|
1023
1027
|
hi = aliases[before]
|
|
1024
1028
|
window = items[lo:hi]
|
|
1025
|
-
if
|
|
1026
|
-
|
|
1027
|
-
window = window[-cap:] if cap else window
|
|
1028
|
-
elif limit:
|
|
1029
|
-
window = window[:limit]
|
|
1029
|
+
if limit:
|
|
1030
|
+
window = window[-limit:] if tail else window[:limit]
|
|
1030
1031
|
first_key = window[0]["item_key"] if window else None
|
|
1031
1032
|
last_key = window[-1]["item_key"] if window else None
|
|
1032
1033
|
has_before = bool(window) and keys.index(first_key) > 0
|
|
@@ -1047,7 +1048,7 @@ def get_codex_conversation(
|
|
|
1047
1048
|
effective_speed: str,
|
|
1048
1049
|
after: str | None = None,
|
|
1049
1050
|
before: str | None = None,
|
|
1050
|
-
tail:
|
|
1051
|
+
tail: bool = False,
|
|
1051
1052
|
limit: int = 200,
|
|
1052
1053
|
legacy_export: bool = False,
|
|
1053
1054
|
) -> dict:
|
package/bin/_lib_conversation.py
CHANGED
|
@@ -171,6 +171,7 @@ def _strip_remote_control_prefix(text):
|
|
|
171
171
|
|
|
172
172
|
|
|
173
173
|
_TOOL_RESULT_CAP = 16000 # was 4000; full text always re-derivable from JSONL
|
|
174
|
+
_BACKGROUND_NOTIFICATION_BLOCK_WIRE_CAP = 230_000
|
|
174
175
|
_INPUT_LEAF_CAP = 8000 # max chars per string leaf in a bounded tool input
|
|
175
176
|
_INPUT_TOTAL_CAP = 32000 # honesty backstop on the serialized bounded input
|
|
176
177
|
_INPUT_MAX_NODES = 2000 # max dict-values + list-elements kept before tail elision
|
|
@@ -281,16 +282,31 @@ def _queued_prompt_row(obj, t, offset):
|
|
|
281
282
|
``{"type":"attachment","attachment":{"type":"queued_command",
|
|
282
283
|
"commandMode":"prompt","prompt":<text>}}`` — carrying its OWN
|
|
283
284
|
uuid/parentUuid/timestamp, with the text in ``attachment.prompt`` rather than
|
|
284
|
-
``message.content``.
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
285
|
+
``message.content``.
|
|
286
|
+
|
|
287
|
+
``commandMode=="task-notification"`` is harness-injected background plumbing
|
|
288
|
+
rather than something the user typed, so it is NOT promoted as a HUMAN turn —
|
|
289
|
+
but it is no longer dropped either. This used to be justified by "the same
|
|
290
|
+
<task-notification> content already classifies META when it arrives as a
|
|
291
|
+
regular line", which is true for ONE notification shape and false for the
|
|
292
|
+
other, and the distinction is a whole bug class:
|
|
293
|
+
|
|
294
|
+
A subagent / Monitor arrives as a type:"user" line, joins on
|
|
295
|
+
<tool-use-id> -> already classified META
|
|
296
|
+
B backgrounded MCP arrives ONLY as this attachment, carries
|
|
297
|
+
<task-id> and NO <tool-use-id> -> nothing
|
|
298
|
+
rescued it, so the response was lost
|
|
299
|
+
|
|
300
|
+
Claude Code moved MCP dispatch to the background after 120s, which made
|
|
301
|
+
shape B common; before that the assumption held in practice. Shape B is now
|
|
302
|
+
promoted as a BOUNDED META row (see ``_background_notification_row``)."""
|
|
289
303
|
if t != "attachment" or not obj.get("uuid"):
|
|
290
304
|
return None
|
|
291
305
|
att = obj.get("attachment")
|
|
292
306
|
if not isinstance(att, dict) or att.get("type") != "queued_command":
|
|
293
307
|
return None
|
|
308
|
+
if att.get("commandMode") == "task-notification":
|
|
309
|
+
return _background_notification_row(obj, att.get("prompt"), offset)
|
|
294
310
|
if att.get("commandMode") != "prompt":
|
|
295
311
|
return None
|
|
296
312
|
prompt = att.get("prompt")
|
|
@@ -308,6 +324,79 @@ def _queued_prompt_row(obj, t, offset):
|
|
|
308
324
|
return _normalize(synth, "user", offset)
|
|
309
325
|
|
|
310
326
|
|
|
327
|
+
def _background_notification_row(obj, prompt, offset):
|
|
328
|
+
"""A backgrounded-MCP <task-notification> -> a BOUNDED META ``MessageRow``.
|
|
329
|
+
|
|
330
|
+
The raw wrapper is deliberately NOT stored verbatim. Text blocks are not
|
|
331
|
+
subject to ``_TOOL_RESULT_CAP`` and read-time meta classification restores
|
|
332
|
+
the whole body, so a verbatim store would put an arbitrary (multi-megabyte)
|
|
333
|
+
MCP payload into the transcript store and ship it whole to the browser
|
|
334
|
+
whenever the read-time join misses. Instead the notification is parsed HERE
|
|
335
|
+
into one bounded text block carrying the identity/status sidecars the
|
|
336
|
+
read-time join and the full-payload resolver consume.
|
|
337
|
+
|
|
338
|
+
``text`` = a synthesized header plus the capped result, so a notification
|
|
339
|
+
whose join does NOT fire still renders something sane through the ordinary
|
|
340
|
+
text path. ``result_offset`` is the EXACT index where the capped result
|
|
341
|
+
begins inside ``text`` (``None`` when there is no result), so the join can
|
|
342
|
+
strip the header without re-deriving it — the card must receive the bare
|
|
343
|
+
``{"threadId":…,"content":…}`` envelope its parser decodes, and a summary
|
|
344
|
+
containing a blank line would defeat a scan-based strip.
|
|
345
|
+
|
|
346
|
+
Classified META with ``text=""`` — like the shape-A notification already is
|
|
347
|
+
— which keeps it out of prose FTS, title derivation and the ``human``
|
|
348
|
+
prompts facet. The capped result still reaches ``search_tool``, so a
|
|
349
|
+
recovered response stays findable through dashboard search, in-conversation
|
|
350
|
+
find and ``transcript search``, exactly as the same result delivered inline
|
|
351
|
+
would be."""
|
|
352
|
+
import _lib_background_mcp as _bg
|
|
353
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
354
|
+
return None
|
|
355
|
+
n = _bg.parse_task_notification(prompt)
|
|
356
|
+
if n is None:
|
|
357
|
+
return None # no <task-id> -> no identity -> nothing to join
|
|
358
|
+
full = n.result_text or ""
|
|
359
|
+
capped = full[:_TOOL_RESULT_CAP]
|
|
360
|
+
header = f"Background task {n.task_id} — {n.status or 'unknown'}"
|
|
361
|
+
if n.summary:
|
|
362
|
+
header += f"\n{n.summary}"
|
|
363
|
+
block = {
|
|
364
|
+
"kind": "text",
|
|
365
|
+
"text": header + (("\n\n" + capped) if capped else ""),
|
|
366
|
+
"task_id": n.task_id,
|
|
367
|
+
"background_status": n.status,
|
|
368
|
+
"summary": n.summary,
|
|
369
|
+
"result_offset": (len(header) + 2) if capped else None,
|
|
370
|
+
"result_full_length": len(full),
|
|
371
|
+
"result_truncated": len(full) > _TOOL_RESULT_CAP,
|
|
372
|
+
}
|
|
373
|
+
blocks = [block]
|
|
374
|
+
blocks_json = json.dumps(blocks, separators=(",", ":"))
|
|
375
|
+
if len(blocks_json) > _BACKGROUND_NOTIFICATION_BLOCK_WIRE_CAP:
|
|
376
|
+
# All copied fields are capped by the background-MCP kernel. Keep a
|
|
377
|
+
# fail-closed storage backstop so a future field cannot silently make
|
|
378
|
+
# this supposedly bounded row unbounded.
|
|
379
|
+
return None
|
|
380
|
+
# Derive through the SAME chokepoint a blocks_json backfill would use, so
|
|
381
|
+
# ingest and any recompute-from-blocks pass agree by construction (#177 S6).
|
|
382
|
+
search_tool, search_thinking = _derive_search_columns(blocks)
|
|
383
|
+
return MessageRow(
|
|
384
|
+
byte_offset=offset,
|
|
385
|
+
session_id=obj.get("sessionId"),
|
|
386
|
+
uuid=obj.get("uuid"),
|
|
387
|
+
parent_uuid=obj.get("parentUuid"),
|
|
388
|
+
timestamp_utc=obj.get("timestamp"),
|
|
389
|
+
entry_type=META,
|
|
390
|
+
text="",
|
|
391
|
+
blocks_json=blocks_json,
|
|
392
|
+
model=None, msg_id=None, req_id=None,
|
|
393
|
+
cwd=obj.get("cwd"), git_branch=obj.get("gitBranch"),
|
|
394
|
+
is_sidechain=1 if obj.get("isSidechain") else 0,
|
|
395
|
+
search_tool=search_tool,
|
|
396
|
+
search_thinking=search_thinking,
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
|
|
311
400
|
@dataclass
|
|
312
401
|
class AiTitleRow:
|
|
313
402
|
"""Pure per-line AI-title record (no I/O). Parallels MessageRow but for the
|
|
@@ -1194,6 +1283,17 @@ def _derive_search_columns(blocks):
|
|
|
1194
1283
|
t = b.get("text") or ""
|
|
1195
1284
|
if t:
|
|
1196
1285
|
think_parts.append(t[:_TOOL_RESULT_CAP])
|
|
1286
|
+
elif k == "text" and b.get("task_id") and b.get("result_offset") is not None:
|
|
1287
|
+
# A recovered backgrounded-MCP result. It is TOOL output that merely
|
|
1288
|
+
# rides in a text block (the block shape a renderer without the join
|
|
1289
|
+
# can still display), so it belongs in search_tool — the identical
|
|
1290
|
+
# result delivered inline is searchable and this must not regress to
|
|
1291
|
+
# silently unsearchable. Gated on the task_id + result_offset pair so
|
|
1292
|
+
# an ordinary prose text block is never indexed here (prose already
|
|
1293
|
+
# lives in the `text` column).
|
|
1294
|
+
res = (b.get("text") or "")[b["result_offset"]:]
|
|
1295
|
+
if res:
|
|
1296
|
+
tool_parts.append(res[:_TOOL_RESULT_CAP])
|
|
1197
1297
|
elif k == "tool_use":
|
|
1198
1298
|
tool_parts.extend(
|
|
1199
1299
|
s[:_TOOL_RESULT_CAP] for s in _aux_strings(b.get("input")))
|
|
@@ -545,7 +545,7 @@ def neutral_detail(
|
|
|
545
545
|
if cref.source == "codex":
|
|
546
546
|
return q.get_codex_conversation(
|
|
547
547
|
conn, cref.conversation_key, effective_speed=speed,
|
|
548
|
-
after=after, before=before, tail=tail,
|
|
548
|
+
after=after, before=before, tail=bool(tail),
|
|
549
549
|
limit=limit if limit is not None else 200)
|
|
550
550
|
return _claude_detail(
|
|
551
551
|
conn, cref.native_key, cref.conversation_key,
|
|
@@ -799,11 +799,22 @@ def neutral_payload(
|
|
|
799
799
|
# Claude: tool_use_id + which={input,result}, contract unchanged.
|
|
800
800
|
if not tool_use_id or which not in ("input", "result"):
|
|
801
801
|
return {"status": "not_found", "tool_use_id": tool_use_id, "which": which}
|
|
802
|
-
|
|
802
|
+
if which == "result":
|
|
803
|
+
# Both payload routes address the SAME logical object, so both resolve it
|
|
804
|
+
# through the same carrier. A naive which="result" lookup finds the
|
|
805
|
+
# backgrounded-MCP PLACEHOLDER row first and would serve "still running
|
|
806
|
+
# after 120s" as the full response — 200, and silently wrong content.
|
|
807
|
+
mode, loc = lcq.locate_result_payload(conn, cref.native_key, tool_use_id)
|
|
808
|
+
if mode == "background_gone":
|
|
809
|
+
# A KNOWN background placeholder whose notification is unresolvable:
|
|
810
|
+
# gone (410), never not_found (404) and never the placeholder text.
|
|
811
|
+
return {"status": "gone", "tool_use_id": tool_use_id, "which": which}
|
|
812
|
+
else:
|
|
813
|
+
mode = which
|
|
814
|
+
loc = lcq.locate_tool_payload(conn, cref.native_key, tool_use_id, which)
|
|
803
815
|
if loc is None:
|
|
804
816
|
return {"status": "not_found", "tool_use_id": tool_use_id, "which": which}
|
|
805
|
-
|
|
806
|
-
payload = lcq.read_full_payload(source_path, byte_offset, tool_use_id, which)
|
|
817
|
+
payload = lcq.read_located_payload(loc, tool_use_id, mode)
|
|
807
818
|
if payload is None:
|
|
808
819
|
return {"status": "gone", "tool_use_id": tool_use_id, "which": which}
|
|
809
820
|
return {"status": "ok", **payload}
|
|
@@ -54,6 +54,11 @@ from _lib_conversation import (
|
|
|
54
54
|
_is_compaction_body, _is_notification_body, _is_bash_echo_body,
|
|
55
55
|
_strip_remote_control_prefix,
|
|
56
56
|
)
|
|
57
|
+
# Backgrounded-MCP recovery (spec 2026-07-31): the placeholder/notification
|
|
58
|
+
# parsers AND the fail-closed selection rule. The finalize-stage join and
|
|
59
|
+
# locate_tool_payload's background_result mode both call select_background_joins
|
|
60
|
+
# so a card can never display one capped response and load a different full one.
|
|
61
|
+
import _lib_background_mcp as _bg
|
|
57
62
|
|
|
58
63
|
# ── Opt-in phase instrumentation (issue #276, Session C / M5) ────────────────
|
|
59
64
|
# This kernel is imported DIRECTLY by several tests (tests/test_conversation_*),
|
|
@@ -1685,6 +1690,7 @@ def _assemble_session(conn, session_id):
|
|
|
1685
1690
|
turn_index = {} # (msg_id, req_id) -> index into items
|
|
1686
1691
|
tooluse_index = {} # tool_use id -> (item, block_dict)
|
|
1687
1692
|
tool_result_items = [] # placeholder items deferred to Phase 2
|
|
1693
|
+
_bg_placeholder_claims_all = [] # canonical pre-fold claimant pairs
|
|
1688
1694
|
|
|
1689
1695
|
def _index_tool_uses(item):
|
|
1690
1696
|
# Index every tool_use id -> its (item, block). Idempotent: re-scanning
|
|
@@ -1714,6 +1720,8 @@ def _assemble_session(conn, session_id):
|
|
|
1714
1720
|
it = _build_simple(row)
|
|
1715
1721
|
items.append(it)
|
|
1716
1722
|
tool_result_items.append(it)
|
|
1723
|
+
_bg_placeholder_claims_all.extend(
|
|
1724
|
+
_bg_placeholder_claims(it["blocks"]))
|
|
1717
1725
|
else:
|
|
1718
1726
|
it = _build_simple(row)
|
|
1719
1727
|
items.append(it)
|
|
@@ -2051,6 +2059,87 @@ def _assemble_session(conn, session_id):
|
|
|
2051
2059
|
if _link is not None and _link[0] in subagent_meta:
|
|
2052
2060
|
subagent_meta[_link[0]]["status"] = _status # upgrades async_launched -> completed
|
|
2053
2061
|
|
|
2062
|
+
# Backgrounded-MCP completion (spec 2026-07-31 §2). The sibling join above
|
|
2063
|
+
# handles shape A — a subagent/Monitor notification carrying <tool-use-id>.
|
|
2064
|
+
# Shape B (an MCP call Claude Code moved to the background after 120s)
|
|
2065
|
+
# carries <task-id> and NO <tool-use-id>, so the bridge is the PLACEHOLDER
|
|
2066
|
+
# tool_result, which names the task twice in two independently worded
|
|
2067
|
+
# places. Runs here, on the UNSLICED item list, so a match can never depend
|
|
2068
|
+
# on page boundaries (get_conversation slices downstream).
|
|
2069
|
+
# Read the canonical PRE-FOLD claims. Phase 2 can fold more than one physical
|
|
2070
|
+
# result into one tool call, and the visible result slot necessarily keeps
|
|
2071
|
+
# only one; collecting there would already have lost conflicting claimants.
|
|
2072
|
+
# The payload resolver independently reconstructs this same canonical-row
|
|
2073
|
+
# population, so capped display and full lookup feed identical claims into
|
|
2074
|
+
# the pure fail-closed selector. Collection rides the existing Phase-1
|
|
2075
|
+
# parse; this finalize stage never reparses the session.
|
|
2076
|
+
_ph_claims = _bg_placeholder_claims_all
|
|
2077
|
+
|
|
2078
|
+
_notifs = [] # BackgroundNotification, document order
|
|
2079
|
+
_notif_src = {} # id(notification) -> (item, block)
|
|
2080
|
+
_notif_by_task = {} # task_id -> [block, …] (status stamping only)
|
|
2081
|
+
for it in items:
|
|
2082
|
+
if it["kind"] != "meta":
|
|
2083
|
+
continue
|
|
2084
|
+
for b in it["blocks"]:
|
|
2085
|
+
# Shared extraction (exact header strip included) so the read-time
|
|
2086
|
+
# join and the full-payload resolver can never diverge.
|
|
2087
|
+
_n = _eligible_bg_notification(it["kind"], it.get("ts"), b)
|
|
2088
|
+
if _n is None:
|
|
2089
|
+
continue
|
|
2090
|
+
_notifs.append(_n)
|
|
2091
|
+
_notif_src[id(_n)] = (it, b)
|
|
2092
|
+
_notif_by_task.setdefault(_n.task_id, []).append(b)
|
|
2093
|
+
|
|
2094
|
+
_joined = _bg.select_background_joins(_ph_claims, _notifs)
|
|
2095
|
+
_bg_drop = set()
|
|
2096
|
+
for _tuid, _n in _joined.items():
|
|
2097
|
+
_hit = tooluse_index.get(_tuid)
|
|
2098
|
+
if _hit is None:
|
|
2099
|
+
continue
|
|
2100
|
+
_owner, _block = _hit
|
|
2101
|
+
_src_item, _src_block = _notif_src[id(_n)]
|
|
2102
|
+
_block["result"] = {
|
|
2103
|
+
"text": _n.result_text,
|
|
2104
|
+
"truncated": bool(_src_block.get("result_truncated")),
|
|
2105
|
+
"full_length": _src_block.get("result_full_length"),
|
|
2106
|
+
"is_error": False,
|
|
2107
|
+
}
|
|
2108
|
+
_block["background_status"] = "completed"
|
|
2109
|
+
_block["background_completed_at"] = _src_item.get("ts")
|
|
2110
|
+
# The Phase-4b fold precedent: the folded row's uuid joins the owner's
|
|
2111
|
+
# member_uuids so jump/permalink ownership, outline mapping and
|
|
2112
|
+
# sidechain-parent lookup all keep resolving.
|
|
2113
|
+
_owner["member_uuids"].append(_src_item["anchor"]["uuid"])
|
|
2114
|
+
_bg_drop.add(id(_src_item))
|
|
2115
|
+
if _bg_drop:
|
|
2116
|
+
items = [it for it in items if id(it) not in _bg_drop]
|
|
2117
|
+
|
|
2118
|
+
# Stamp the status on every call we did NOT recover, so the client renders an
|
|
2119
|
+
# in-flight state rather than a false "ok" (CodexCard reports '✓ ok' for any
|
|
2120
|
+
# non-null non-error result). These notifications stay VISIBLE — a
|
|
2121
|
+
# non-completed one is the only remaining evidence of what happened.
|
|
2122
|
+
_unambiguous_tasks = _bg.unambiguous_placeholder_tasks(_ph_claims)
|
|
2123
|
+
for _tuid in dict.fromkeys(tuid for tuid, _task in _ph_claims):
|
|
2124
|
+
if _tuid in _joined:
|
|
2125
|
+
continue
|
|
2126
|
+
_hit = tooluse_index.get(_tuid)
|
|
2127
|
+
if _hit is None:
|
|
2128
|
+
continue
|
|
2129
|
+
_task = _unambiguous_tasks.get(_tuid)
|
|
2130
|
+
if _task is None:
|
|
2131
|
+
# Conflicting task claims cannot choose a notification or status,
|
|
2132
|
+
# but this is still visibly a background placeholder. Keep the
|
|
2133
|
+
# client in its honest pending state instead of falling through to
|
|
2134
|
+
# the generic non-error "ok" rendering.
|
|
2135
|
+
_hit[1]["background_status"] = "running"
|
|
2136
|
+
continue
|
|
2137
|
+
_cands = _notif_by_task.get(_task) or []
|
|
2138
|
+
_statuses = {c.get("background_status") for c in _cands
|
|
2139
|
+
if c.get("background_status")}
|
|
2140
|
+
_hit[1]["background_status"] = (
|
|
2141
|
+
_statuses.pop() if len(_statuses) == 1 else "running")
|
|
2142
|
+
|
|
2054
2143
|
# Derived totals: any child still missing a count gets it from its own
|
|
2055
2144
|
# subagent_key bucket. tool-count = tool_call/tool_use blocks; duration =
|
|
2056
2145
|
# (max_ts - min_ts) ms; tokens = sum of per-turn token totals (now stamped).
|
|
@@ -3922,11 +4011,170 @@ def _manual_snippet(text, q, width=80):
|
|
|
3922
4011
|
_FULL_PAYLOAD_CEILING = 1_000_000 # serve up to ~1 MB; protects the HTTP server / browser
|
|
3923
4012
|
|
|
3924
4013
|
|
|
4014
|
+
def _bg_notification_from_block(b):
|
|
4015
|
+
"""``BackgroundNotification`` for a stored background-notification block,
|
|
4016
|
+
else ``None``.
|
|
4017
|
+
|
|
4018
|
+
ONE extraction, shared by the finalize-stage join and the full-payload
|
|
4019
|
+
resolver — including the exact header strip: the stored text is
|
|
4020
|
+
``header\\n\\nresult`` and ``result_offset`` is where the result begins, so
|
|
4021
|
+
the card always receives the bare ``{"threadId":…,"content":…}`` envelope its
|
|
4022
|
+
parser decodes."""
|
|
4023
|
+
if not isinstance(b, dict):
|
|
4024
|
+
return None
|
|
4025
|
+
task = b.get("task_id")
|
|
4026
|
+
if not task:
|
|
4027
|
+
return None
|
|
4028
|
+
off = b.get("result_offset")
|
|
4029
|
+
res = (b.get("text") or "")[off:] if isinstance(off, int) else None
|
|
4030
|
+
return _bg.BackgroundNotification(
|
|
4031
|
+
task, b.get("background_status") or "", b.get("summary") or "",
|
|
4032
|
+
res or None)
|
|
4033
|
+
|
|
4034
|
+
|
|
4035
|
+
def _bg_placeholder_claims(blocks):
|
|
4036
|
+
"""Every background placeholder claim in one canonical stored row."""
|
|
4037
|
+
out = []
|
|
4038
|
+
if not isinstance(blocks, list):
|
|
4039
|
+
return out
|
|
4040
|
+
for block in blocks:
|
|
4041
|
+
if not isinstance(block, dict) or block.get("kind") != "tool_result":
|
|
4042
|
+
continue
|
|
4043
|
+
task_id = _bg.parse_placeholder_task_id(block.get("text") or "")
|
|
4044
|
+
tool_use_id = block.get("tool_use_id")
|
|
4045
|
+
if task_id and tool_use_id is not None:
|
|
4046
|
+
out.append((tool_use_id, task_id))
|
|
4047
|
+
return out
|
|
4048
|
+
|
|
4049
|
+
|
|
4050
|
+
def _eligible_bg_notification(item_kind, timestamp, block):
|
|
4051
|
+
"""Assembly/payload shared eligibility for a stored notification row."""
|
|
4052
|
+
if item_kind != "meta" or not isinstance(timestamp, str) or not timestamp:
|
|
4053
|
+
return None
|
|
4054
|
+
return _bg_notification_from_block(block)
|
|
4055
|
+
|
|
4056
|
+
|
|
4057
|
+
# Cheap SQL prefilters for the two block shapes the background scan cares about.
|
|
4058
|
+
# Both are literal substrings of what must be present, so they can only
|
|
4059
|
+
# over-select — a false negative would break the fail-closed guarantee by hiding
|
|
4060
|
+
# a second claimant of the same task id.
|
|
4061
|
+
_BG_PLACEHOLDER_PROBE = "is still running after "
|
|
4062
|
+
_BG_TASK_ID_PROBE = '"task_id":'
|
|
4063
|
+
|
|
4064
|
+
|
|
4065
|
+
def _scan_background_session(conn, session_id):
|
|
4066
|
+
"""``(placeholders, notifications, {id(notification): (source_path, offset)})``
|
|
4067
|
+
over one session's LOGICAL messages.
|
|
4068
|
+
|
|
4069
|
+
Reproduces ``_assemble_session``'s view rather than approximating it: the
|
|
4070
|
+
physical rows are deduped by uuid keeping the earliest ``(timestamp_utc,
|
|
4071
|
+
id)``, exactly as assembly does, so the full-payload path and the rendered
|
|
4072
|
+
card can never disagree about which notification won. The canonical-row
|
|
4073
|
+
decision is made over the WHOLE session (an id/uuid-only read) before the
|
|
4074
|
+
content prefilter is applied — deciding it inside the prefiltered set would
|
|
4075
|
+
let a later physical row become canonical for a uuid whose earliest row
|
|
4076
|
+
assembly actually chose."""
|
|
4077
|
+
canonical = {}
|
|
4078
|
+
for rid, uuid in conn.execute(
|
|
4079
|
+
"SELECT id, uuid FROM conversation_messages WHERE session_id=? "
|
|
4080
|
+
"ORDER BY timestamp_utc, id", (session_id,)
|
|
4081
|
+
):
|
|
4082
|
+
if uuid not in canonical:
|
|
4083
|
+
canonical[uuid] = rid
|
|
4084
|
+
keep = set(canonical.values())
|
|
4085
|
+
|
|
4086
|
+
placeholders, notifications, loc_by_notif = [], [], {}
|
|
4087
|
+
for rid, source_path, byte_offset, entry_type, timestamp, blocks_json in conn.execute(
|
|
4088
|
+
"SELECT id, source_path, byte_offset, entry_type, timestamp_utc, blocks_json "
|
|
4089
|
+
"FROM conversation_messages WHERE session_id=? "
|
|
4090
|
+
"AND (instr(blocks_json, ?) > 0 OR instr(blocks_json, ?) > 0) "
|
|
4091
|
+
"ORDER BY timestamp_utc, id",
|
|
4092
|
+
(session_id, _BG_PLACEHOLDER_PROBE, _BG_TASK_ID_PROBE)
|
|
4093
|
+
):
|
|
4094
|
+
if rid not in keep:
|
|
4095
|
+
continue
|
|
4096
|
+
try:
|
|
4097
|
+
blocks = _json.loads(blocks_json)
|
|
4098
|
+
except (ValueError, TypeError):
|
|
4099
|
+
continue
|
|
4100
|
+
if not isinstance(blocks, list):
|
|
4101
|
+
continue
|
|
4102
|
+
placeholders.extend(_bg_placeholder_claims(blocks))
|
|
4103
|
+
for b in blocks:
|
|
4104
|
+
if not isinstance(b, dict):
|
|
4105
|
+
continue
|
|
4106
|
+
if b.get("kind") == "tool_result":
|
|
4107
|
+
continue
|
|
4108
|
+
n = _eligible_bg_notification(entry_type, timestamp, b)
|
|
4109
|
+
if n is not None:
|
|
4110
|
+
notifications.append(n)
|
|
4111
|
+
loc_by_notif[id(n)] = (source_path, byte_offset)
|
|
4112
|
+
return placeholders, notifications, loc_by_notif
|
|
4113
|
+
|
|
4114
|
+
|
|
4115
|
+
def locate_result_payload(conn, session_id, tool_use_id):
|
|
4116
|
+
"""``(mode, location)`` for a PUBLIC ``which='result'`` request.
|
|
4117
|
+
|
|
4118
|
+
``mode`` is one of:
|
|
4119
|
+
|
|
4120
|
+
- ``'result'`` — an ordinary tool_result; ``location`` is
|
|
4121
|
+
``locate_tool_payload(..., 'result')`` (``None`` for an unknown id -> 404).
|
|
4122
|
+
- ``'background_result'`` — the id is a backgrounded-MCP placeholder whose
|
|
4123
|
+
completion notification resolves unambiguously; ``location`` is a THREE
|
|
4124
|
+
-tuple ``(source_path, byte_offset, task_id)`` addressing the
|
|
4125
|
+
NOTIFICATION's JSONL line. The task id rides along because the re-read at
|
|
4126
|
+
that offset must prove it landed on THIS task's notification.
|
|
4127
|
+
- ``'background_gone'`` — a KNOWN background placeholder whose notification
|
|
4128
|
+
cannot be resolved (deleted, rotated, still running, or ambiguous). The
|
|
4129
|
+
handler maps this to 410, distinct from the 404 an unknown id returns.
|
|
4130
|
+
|
|
4131
|
+
``background_result`` is INTERNAL: the response ``read_full_payload`` builds
|
|
4132
|
+
keeps the public ``which: "result"`` discriminant, so neither the client
|
|
4133
|
+
contract nor the endpoint's accepted input surface changes."""
|
|
4134
|
+
has_background_placeholder = conn.execute(
|
|
4135
|
+
"SELECT 1 FROM conversation_messages WHERE session_id=? "
|
|
4136
|
+
"AND instr(blocks_json, ?) > 0 LIMIT 1",
|
|
4137
|
+
(session_id, _BG_PLACEHOLDER_PROBE),
|
|
4138
|
+
).fetchone()
|
|
4139
|
+
if has_background_placeholder is None:
|
|
4140
|
+
return ("result", locate_tool_payload(conn, session_id, tool_use_id,
|
|
4141
|
+
"result"))
|
|
4142
|
+
placeholders, notifications, loc_by_notif = _scan_background_session(
|
|
4143
|
+
conn, session_id)
|
|
4144
|
+
if not any(tuid == tool_use_id for tuid, _task_id in placeholders):
|
|
4145
|
+
return ("result", locate_tool_payload(conn, session_id, tool_use_id,
|
|
4146
|
+
"result"))
|
|
4147
|
+
# The SAME fail-closed rule assembly applied — called, never re-derived — so
|
|
4148
|
+
# a card showing the placeholder can't load somebody else's full response.
|
|
4149
|
+
n = _bg.select_background_joins(placeholders, notifications).get(tool_use_id)
|
|
4150
|
+
if n is None:
|
|
4151
|
+
return ("background_gone", None)
|
|
4152
|
+
return ("background_result", loc_by_notif[id(n)] + (n.task_id,))
|
|
4153
|
+
|
|
4154
|
+
|
|
4155
|
+
def read_located_payload(loc, tool_use_id, which):
|
|
4156
|
+
"""``read_full_payload`` for a location a locator produced.
|
|
4157
|
+
|
|
4158
|
+
The ``background_result`` location is a 3-tuple whose third element is the
|
|
4159
|
+
task id the re-read line must carry; every other mode is a plain
|
|
4160
|
+
``(source_path, byte_offset)``. Unpacking lives here so both payload routes
|
|
4161
|
+
stay one call and neither can forget the identity check."""
|
|
4162
|
+
return read_full_payload(
|
|
4163
|
+
loc[0], loc[1], tool_use_id, which,
|
|
4164
|
+
expected_task_id=(loc[2] if len(loc) > 2 else None))
|
|
4165
|
+
|
|
4166
|
+
|
|
3925
4167
|
def locate_tool_payload(conn, session_id, tool_use_id, which):
|
|
3926
4168
|
"""``(source_path, byte_offset)`` for the JSONL line holding the tool_use
|
|
3927
4169
|
(``which='input'``) or tool_result (``which='result'``) carrying this
|
|
3928
4170
|
``tool_use_id`` in this session, else ``None``.
|
|
3929
4171
|
|
|
4172
|
+
``which='background_result'`` is the internal backgrounded-MCP mode: it
|
|
4173
|
+
resolves the placeholder for ``tool_use_id``, reads its task id, and
|
|
4174
|
+
addresses the winning NOTIFICATION row instead — returning the 3-tuple
|
|
4175
|
+
``(source_path, byte_offset, task_id)`` that mode carries (see
|
|
4176
|
+
``locate_result_payload``).
|
|
4177
|
+
|
|
3930
4178
|
The prefilter uses ``instr(blocks_json, ?) > 0`` — NOT ``LIKE`` (Codex P1.4):
|
|
3931
4179
|
tool_use_ids contain ``_`` (e.g. ``toolu_01SEQ…``), which ``LIKE`` treats as a
|
|
3932
4180
|
single-char wildcard, so a near-miss id would false-match. ``instr`` is a
|
|
@@ -3937,6 +4185,9 @@ def locate_tool_payload(conn, session_id, tool_use_id, which):
|
|
|
3937
4185
|
same deterministic ``ORDER BY timestamp_utc, id`` as ``get_conversation``. The
|
|
3938
4186
|
SELECT runs here (not via ``get_conversation``) because that reader omits
|
|
3939
4187
|
``byte_offset`` (Codex P2.5)."""
|
|
4188
|
+
if which == "background_result":
|
|
4189
|
+
mode, loc = locate_result_payload(conn, session_id, tool_use_id)
|
|
4190
|
+
return loc if mode == "background_result" else None
|
|
3940
4191
|
rows = conn.execute(
|
|
3941
4192
|
"SELECT source_path, byte_offset, blocks_json FROM conversation_messages "
|
|
3942
4193
|
"WHERE session_id=? AND instr(blocks_json, ?) > 0 "
|
|
@@ -3959,7 +4210,8 @@ def locate_tool_payload(conn, session_id, tool_use_id, which):
|
|
|
3959
4210
|
return None
|
|
3960
4211
|
|
|
3961
4212
|
|
|
3962
|
-
def read_full_payload(source_path, byte_offset, tool_use_id, which
|
|
4213
|
+
def read_full_payload(source_path, byte_offset, tool_use_id, which, *,
|
|
4214
|
+
expected_task_id=None):
|
|
3963
4215
|
"""Re-read the raw JSONL line at ``(source_path, byte_offset)`` and return the
|
|
3964
4216
|
FULL (un-capped) payload for ``tool_use_id``:
|
|
3965
4217
|
|
|
@@ -3974,7 +4226,23 @@ def read_full_payload(source_path, byte_offset, tool_use_id, which):
|
|
|
3974
4226
|
JSONL — the documented 410 path) or the id is no longer present in that line.
|
|
3975
4227
|
``full_length``/``truncated`` describe the payload against ``_FULL_PAYLOAD_CEILING``
|
|
3976
4228
|
— honoring #178's "un-capped" spirit for real payloads while bounding the
|
|
3977
|
-
degenerate multi-MB case.
|
|
4229
|
+
degenerate multi-MB case.
|
|
4230
|
+
|
|
4231
|
+
``expected_task_id`` is REQUIRED by ``which='background_result'`` and ignored
|
|
4232
|
+
otherwise. That line carries no ``tool_use_id`` to match on, so the task id is
|
|
4233
|
+
the only identity it has: without the check, a rotated file whose stored
|
|
4234
|
+
offset now lands on a DIFFERENT task-notification would serve a foreign
|
|
4235
|
+
result under this call's id. A MISMATCH -> ``None`` (the 410 path), the same
|
|
4236
|
+
fail-closed shape the ``result``/``input`` branches get from their exact id
|
|
4237
|
+
match. An ABSENT one is different in kind: it is a wiring mistake, not a
|
|
4238
|
+
rotation, and it would fail closed for EVERY recovered background result
|
|
4239
|
+
while being indistinguishable in a log from the genuine article — so it
|
|
4240
|
+
raises ``TypeError`` instead of degrading silently."""
|
|
4241
|
+
if which == "background_result" and not expected_task_id:
|
|
4242
|
+
raise TypeError(
|
|
4243
|
+
"read_full_payload(which='background_result') requires a non-empty "
|
|
4244
|
+
"expected_task_id — the notification line has no tool_use_id to "
|
|
4245
|
+
"match on, so the task id is its only identity")
|
|
3978
4246
|
try:
|
|
3979
4247
|
with open(source_path, "rb") as fh:
|
|
3980
4248
|
fh.seek(byte_offset)
|
|
@@ -3984,6 +4252,30 @@ def read_full_payload(source_path, byte_offset, tool_use_id, which):
|
|
|
3984
4252
|
return None
|
|
3985
4253
|
if not isinstance(obj, dict):
|
|
3986
4254
|
return None
|
|
4255
|
+
if which == "background_result":
|
|
4256
|
+
# A backgrounded-MCP completion is an attachment record: it has NO
|
|
4257
|
+
# ``message.content``, which is exactly why the public which='result'
|
|
4258
|
+
# path could never serve it. Re-extract <result> from the raw wrapper.
|
|
4259
|
+
# ``tool_use_id`` is not present in this line at all — it is echoed back
|
|
4260
|
+
# only because the client keyed its request by it, so the TASK ID is the
|
|
4261
|
+
# only identity available and it is checked below.
|
|
4262
|
+
att = obj.get("attachment")
|
|
4263
|
+
if not isinstance(att, dict):
|
|
4264
|
+
return None
|
|
4265
|
+
n = _bg.parse_task_notification(att.get("prompt"))
|
|
4266
|
+
if n is None or not n.result_text:
|
|
4267
|
+
return None # rotated onto a non-notification line -> 410
|
|
4268
|
+
if n.task_id != expected_task_id:
|
|
4269
|
+
# Rotated onto a DIFFERENT task's notification: well-formed, with a
|
|
4270
|
+
# well-formed <result>, and NOT ours. Serving it would attach a
|
|
4271
|
+
# foreign response to this call. (An absent id never reaches here —
|
|
4272
|
+
# it raised at the top as the wiring bug it is.)
|
|
4273
|
+
return None
|
|
4274
|
+
raw = n.result_text
|
|
4275
|
+
return {"which": "result", "tool_use_id": tool_use_id,
|
|
4276
|
+
"text": raw[:_FULL_PAYLOAD_CEILING], "full_length": len(raw),
|
|
4277
|
+
"truncated": len(raw) > _FULL_PAYLOAD_CEILING,
|
|
4278
|
+
"is_error": False}
|
|
3987
4279
|
content = (obj.get("message") or {}).get("content")
|
|
3988
4280
|
if not isinstance(content, list):
|
|
3989
4281
|
return None
|