cctally 1.53.0 → 1.55.0

package/bin/_lib_conversation_query.py

@@ -287,6 +287,103 @@ def _subagent_key(source_path):
 from _lib_conversation import (
     _NESTED_AGENT_ID_RE, _NESTED_USAGE_RE, _nested_agent_stamp_from_text,
 )
+# #217 S5 F2 — the true per-edit +N/-M stat, recomputed at outline-build time for
+# non-truncated edits (the stamped `block["edit_stat"]` only exists on truncated
+# inputs, where the bounded copy can't be recounted; Codex P1-7).
+from _lib_conversation import _edit_stat_for
+
+# #217 S5 F2 — the files-touched aggregation is DELIBERATELY scoped to the three
+# named edit tools, NOT `_lib_conversation._FILE_TOUCH_TOOLS` (which also includes
+# NotebookEdit — rare; out of scope, Codex P2-4). Lower-cased for case-insensitive
+# block-name matching.
+_EXPORT_EDIT_TOOLS = {"edit", "multiedit", "write"}
+
+
+def _collect_files(items):
+    """Whole-session files-touched aggregation (#217 S5 F2, spec §2).
+
+    For every Edit / MultiEdit / Write ``tool_call`` block (NOT NotebookEdit —
+    the narrower set is deliberate, Codex P2-4) carrying a string ``file_path``,
+    emit one ``touch`` per call and a per-path running ``+N/-M`` sum, preserving
+    first-touch document order. The signed stat is the stamped ``edit_stat`` when
+    present (truncated inputs), else a recompute via ``_edit_stat_for``; when
+    NEITHER is available the touch is STILL listed with ``add``/``del`` = ``None``
+    and the file total sums only the known values (``None`` only when every touch
+    is unknown — Codex P1-7). Pure over the assembled ``items``.
+    """
+    order, agg = [], {}
+    for it in items:
+        anchor_uuid = it["anchor"]["uuid"]
+        for b in it.get("blocks", []):
+            if b.get("kind") not in ("tool_call", "tool_use"):
+                continue
+            nm = (b.get("name") or "").lower()
+            if nm not in _EXPORT_EDIT_TOOLS:
+                continue
+            inp = b.get("input")
+            path = inp.get("file_path") if isinstance(inp, dict) else None
+            if not isinstance(path, str) or not path:
+                continue
+            st = b.get("edit_stat")
+            if st is None:
+                st = _edit_stat_for(b.get("name"), inp)
+            add = st.get("add") if isinstance(st, dict) else None
+            dele = st.get("del") if isinstance(st, dict) else None
+            if path not in agg:
+                agg[path] = {"path": path, "add": None, "del": None, "touches": []}
+                order.append(path)
+            entry = agg[path]
+            entry["touches"].append({
+                "uuid": anchor_uuid,
+                "tool_use_id": b.get("tool_use_id") or b.get("id"),
+                "op": nm,
+                "add": add,
+                "del": dele,
+            })
+            if add is not None:
+                entry["add"] = (entry["add"] or 0) + add
+            if dele is not None:
+                entry["del"] = (entry["del"] or 0) + dele
+    return [agg[p] for p in order]
+
+
+def _task_completion(items):
+    """Whole-session task-completion detection (#217 S5 F7, spec §3).
+
+    Scan the assembled ``items`` in document order, MAIN THREAD ONLY
+    (``subagent_key is None`` AND not ``is_sidechain`` — Codex P1-1, because
+    ``_fold_task_runs`` keeps task state PER subagent thread, so a global
+    last-snapshot could describe a subagent's checklist rather than the
+    session's). Take the LAST main-thread block carrying a folded
+    ``task_snapshot`` (the Task* family, stamped by ``_fold_task_runs``) or a
+    legacy ``TodoWrite`` (``name == "TodoWrite"``, ``input.todos``). Returns
+    ``{all_done, total, completed, anchor_uuid}`` or ``None`` when the
+    main thread has no tasks. Pure over the assembled ``items``.
+    """
+    final = None
+    final_uuid = None
+    for it in items:
+        if it.get("subagent_key") is not None or it.get("is_sidechain"):
+            continue
+        for b in it.get("blocks", []):
+            if b.get("kind") not in ("tool_call", "tool_use"):
+                continue
+            snap = b.get("task_snapshot")
+            if snap is None and b.get("name") == "TodoWrite":
+                inp = b.get("input")
+                snap = inp.get("todos") if isinstance(inp, dict) else None
+            if isinstance(snap, list):
+                final = snap
+                final_uuid = it["anchor"]["uuid"]
+    if not final:
+        return None
+    total = len(final)
+    completed = sum(1 for t in final
+                    if isinstance(t, dict) and t.get("status") == "completed")
+    return {"all_done": total > 0 and completed == total,
+            "total": total,
+            "completed": completed,
+            "anchor_uuid": final_uuid}
 
 
 def _parse_nested_agent_result(text):
@@ -1126,6 +1223,13 @@ def _turn_usage_map(conn, turn_keys):
     return usage
 
 
+# #225: the two tool names that spawn a subagent. `Agent` = modern, `Task` =
+# older. Exact-match (does NOT match the TaskCreate/TaskUpdate/TaskList checklist
+# family). A spawn is identified by this name even when it carried no
+# subagent_type (the default general-purpose dispatch).
+_SPAWN_TOOL_NAMES = ("Agent", "Task")
+
+
 def _assemble_session(conn, session_id):
     """Shared assembly for get_conversation / get_conversation_outline (#177 S5).
 
@@ -1223,12 +1327,15 @@ def _assemble_session(conn, session_id):
     # tool_call/tool_result block shapes are unchanged — the only new output is
     # the top-level subagent_meta map (no undocumented block keys leak). Join is
     # spawn tool_use id <-> tool_result tool_use_id; agent_id == subagent_key.
-    spawn_kind = {}     # tool_use id -> subagent_type
-    spawn_desc = {}     # tool_use id -> spawning Task description (#193)
+    spawn_kind = {}     # tool_use id -> subagent_type (kind), only when declared
+    spawn_ids = {}      # tool_use id -> None: ordered set of Agent/Task spawn ids
+                        # (#225, typed OR untyped). spawn_kind ⊆ spawn_ids. Dict
+                        # (NOT a set) so iteration stays document-ordered.
+    spawn_desc = {}     # tool_use id -> spawning Agent/Task description (#193/#225)
     agent_link = {}     # tool_use id -> (agent_id, raw_meta)
     nested_candidates = []  # §4 1a: (tool_use_id, block) — string-content spawn
                             # results with no structured agent_id, resolved after
-                            # spawn_kind is complete (text still present pre-fold)
+                            # the spawn maps are complete (text still present pre-fold)
     ask_link = {}       # tool_use id -> (answers, annotations)  (#177 S2)
     bash_link = {}      # tool_use id -> (stderr, interrupted)   (#177 S3)
     web_search_link = {}  # tool_use id -> web_search payload    (#177 S4)
@@ -1239,15 +1346,23 @@ def _assemble_session(conn, session_id):
             k = b.get("kind")
             if k == "tool_use":
                 st = b.pop("subagent_type", None)
-                if st and b.get("id") is not None:
-                    spawn_kind[b["id"]] = st
-                    # #193: harvest the spawning Task description from the
-                    # already-stored bounded input. Guarded on subagent_type, so
-                    # a Bash `description` (no subagent_type) is NEVER picked up.
+                _id = b.get("id")
+                # #225: a spawn is an Agent/Task tool_use. Identify it by tool
+                # NAME (so an untyped general-purpose spawn — no subagent_type —
+                # still counts) OR by a recorded subagent_type. The name gate is
+                # what keeps a Bash/Read `description` (and a coincidental
+                # structured agent_id, #218 I-B.5) from ever being read as a
+                # spawn.
+                if _id is not None and (st or b.get("name") in _SPAWN_TOOL_NAMES):
+                    spawn_ids[_id] = None
+                    if st:
+                        spawn_kind[_id] = st       # #166: declared kind
+                    # #193/#225: harvest the spawn description from the bounded
+                    # input for ANY spawn (typed or untyped); name-gated above.
                     _inp = b.get("input")
                     _d = _inp.get("description") if isinstance(_inp, dict) else None
                     if isinstance(_d, str) and _d.strip():
-                        spawn_desc[b["id"]] = _d
+                        spawn_desc[_id] = _d
             elif k == "tool_result":
                 aid = b.pop("agent_id", None)
                 meta = b.pop("subagent_meta", None)
@@ -1255,7 +1370,7 @@ def _assemble_session(conn, session_id):
                     agent_link[b["tool_use_id"]] = (aid, meta or {})
                 elif b.get("tool_use_id") is not None:
                     # §4 1a candidate — a tool_result with no structured agentId.
-                    # Resolved AFTER spawn_kind is complete (text still present,
+                    # Resolved AFTER the spawn maps are complete (text still present,
                     # pre-Phase-2-fold). The block's `text` holds the result body.
                     nested_candidates.append((b["tool_use_id"], b))
                 ans = b.pop("ask_answers", None)            # #177 S2
@@ -1276,30 +1391,33 @@ def _assemble_session(conn, session_id):
                 tlist_ = b.pop("task_list", None)
                 if b.get("tool_use_id") is not None and (tid_ is not None or tlist_ is not None):
                     task_link[b["tool_use_id"]] = {"task_id": tid_, "task_list": tlist_}
-    # §4 symmetry (#218 I-B.5): the structured-agent_id populate above runs
-    # mid-loop, before spawn_kind is complete, so it cannot self-gate on spawn
-    # ownership. Re-validate here now that spawn_kind is final — drop any
-    # agent_link entry whose owning tool_use is NOT a spawn, mirroring the nested
-    # fallback's `_tuid in spawn_kind` gate so BOTH link paths only ever carry
-    # spawn-owned results. Inert today (every agent_link consumer is itself
-    # spawn-gated); this protects a future, un-gated agent_link consumer.
-    agent_link = {t: v for t, v in agent_link.items() if t in spawn_kind}
+    # §4 symmetry (#218 I-B.5 / #225): drop any agent_link entry whose owning
+    # tool_use is NOT a spawn — an ordinary tool result that coincidentally
+    # carries a structured agent_id (e.g. a Read). Gated on spawn_ids (every
+    # Agent/Task spawn, typed OR untyped) rather than the old spawn_kind, which
+    # wrongly excluded untyped general-purpose spawns. A structured agent_id on a
+    # real spawn is authoritative; the heuristic nested text-parse below stays
+    # spawn-gated all the same.
+    agent_link = {t: v for t, v in agent_link.items() if t in spawn_ids}
     # §4 1a — nested grandchild results: gate the text-parse on the owning
-    # tool_use being a spawn (in spawn_kind), so an ordinary tool result that
+    # tool_use being a spawn (in spawn_ids), so an ordinary tool result that
     # merely mentions "agentId" never matches.
     for _tuid, _block in nested_candidates:
-        if _tuid in spawn_kind and _tuid not in agent_link:
+        if _tuid in spawn_ids and _tuid not in agent_link:
             parsed = _parse_nested_agent_result(_block.get("text"))
             if parsed is not None:
                 agent_link[_tuid] = parsed
     subagent_meta = {}
-    for _tuid, _kind in spawn_kind.items():
+    for _tuid in spawn_ids:                 # document order (ordered dict)
         _link = agent_link.get(_tuid)
         if _link is None:
             continue                       # spawn with no (yet) result -> title-only
         _aid, _raw = _link
-        _entry = {"kind": _kind}
-        if spawn_desc.get(_tuid):          # #193: spawning Task description
+        # #225: kind defaults to "general-purpose" for an untyped Agent/Task
+        # spawn (subagent_type omitted at dispatch ⟹ general-purpose; 0
+        # counterexamples across 3085 real sidecars).
+        _entry = {"kind": spawn_kind.get(_tuid, "general-purpose")}
+        if spawn_desc.get(_tuid):          # #193/#225: spawn description
             _entry["description"] = spawn_desc[_tuid]
         for _f in ("total_tokens", "total_duration_ms", "total_tool_use_count", "status"):
             if _raw.get(_f) is not None:
@@ -1869,9 +1987,60 @@ def get_conversation_outline(conn, session_id):
             # matching the per-item / header cost display precision.
             "subagent_costs": {k: round(v, 6) for k, v in subagent_costs.items()},
             "stats": stats,
+            # #217 S5 F2 — whole-session files-touched aggregation (always
+            # present, possibly []; additive — existing consumers are byte-stable).
+            "files": _collect_files(items),
+            # #217 S5 F7 — main-thread task-completion (always present; None when
+            # the main thread has no tasks; additive — byte-stable for consumers).
+            "task_completion": _task_completion(items),
             "turns": turns}
 
 
+def get_conversation_prompts(conn, session_id):
+    """``{session_id, prompts:[{uuid,text}]}`` — ordered main-thread human
+    prompts with full text (#217 S7, F10 — the session-comparison spine).
+
+    Reuses the SAME ``_assemble_session`` pass as ``get_conversation_outline``
+    (one grouping/dedup pipeline, never a parallel one) and the SAME main-thread
+    predicate as the recipe/prompts export (``extract_prompt_entries`` →
+    ``_human_prompts``), so the spine aligns 1:1 with the outline's human turns
+    and can never drift from the export. Returns ``None`` for an unknown session
+    (the handler's 404 sentinel) — the same contract ``get_conversation_outline``
+    uses (``asm is None``).
+    """
+    asm = _assemble_session(conn, session_id)
+    if asm is None:
+        return None
+    from _lib_conversation_export import extract_prompt_entries
+    return {"session_id": session_id,
+            "prompts": extract_prompt_entries(asm["items"])}
+
+
+def get_conversation_export(conn, session_id, scope):
+    """Whole-session Markdown export for one scope (#217 S5, F1/F5).
+
+    Runs the SAME full ``_assemble_session`` the reader/outline use, then hands
+    the assembled ``items`` (+ ``subagent_meta``) to the pure
+    ``export_session_markdown`` serializer (`_lib_conversation_export.py`), so
+    the renderer stays I/O-free and golden-testable. Returns the Markdown string,
+    or ``None`` for an unknown session (the handler's 404 sentinel). ``scope`` is
+    pre-validated by the handler (an unknown scope never reaches here)."""
+    from _lib_conversation_export import export_session_markdown
+    asm = _assemble_session(conn, session_id)
+    if asm is None:
+        return None
+    items = asm["items"]
+    subagent_meta = asm["subagent_meta"]
+    logical = asm["logical"]
+    # Title via the SAME fallback chain the reader header uses (#193):
+    # ai-title → first human prompt → project label → session_id.
+    _pl = _project_label(_latest(logical, 10))
+    title = (_session_titles_map(conn, [session_id]).get(session_id)
+             or _pl or session_id)
+    return export_session_markdown(items, scope, subagent_meta=subagent_meta,
+                                   title=title, session_id=session_id)
+
+
 _TASK_TRIO = ("TaskCreate", "TaskUpdate", "TaskList")
 
 
@@ -2272,8 +2441,8 @@ def _like_escape(q):
     one) mis-escapes any appended ``%`` and the LIKE silently matches nothing.
 
     Single source of the escape chain (#217 S2 / I-3 review Minor #3):
-    ``_like_pattern`` (substring) and ``_search_files``'s prefix pattern both build
-    on this so the escaping is defined exactly once."""
+    ``_like_pattern`` (substring) and ``_search_files``'s substring pattern both
+    build on this so the escaping is defined exactly once."""
     return q.replace("\\", "\\\\").replace("%", r"\%").replace("_", r"\_")
 
 
@@ -2664,13 +2833,6 @@ def _search_title(conn, q, limit, offset, fts_available, filt_sql="", filt_param
             "total": total}
 
 
-# #217 S2 / I-3: a path query "looks like a prefix" when it carries no leading
-# path separator — ``bin/cctally`` is a prefix probe, ``/dashboard`` (or a query
-# that intentionally starts mid-path) forces the substring scan. The set is the
-# common path separators (POSIX + Windows).
-_PATH_SEPARATORS = ("/", "\\")
-
-
 def _search_files(conn, q, limit, offset, filt_sql="", filt_params=()):
     """#217 S2 / I-3: ``kind=files`` cross-session search over the write-class
     file-touch axis (``conversation_file_touches``). Emits one hit per DISTINCT
@@ -2679,28 +2841,22 @@ def _search_files(conn, q, limit, offset, filt_sql="", filt_params=()):
     ``match_kinds:["file"]``, snippet = the file path. ``total`` = the distinct
     ``(session_id, file_path)`` count.
 
-    P3-10 (match shape): a PREFIX-looking query (no leading path separator) binds a
-    literal-prefix pattern (escaped(q) + ``%``) into ``file_path LIKE ? ESCAPE '\\'``
-    and is GENUINELY index-assisted — it rides ``idx_file_touches_path`` as a SEARCH
-    over a key range (verified by EXPLAIN QUERY PLAN in
-    ``test_kind_files_prefix_branch_uses_index_substring_scans``). That assistance
-    depends on ``idx_file_touches_path`` being ``COLLATE NOCASE``: the default
-    ``LIKE`` is case-insensitive, and SQLite only optimizes ``LIKE 'prefix%'`` onto
-    a btree when the index folds the same way (a BINARY index leaves it a full
-    SCAN). The ``ESCAPE '\\'`` clause does NOT defeat the optimization — the literal
-    prefix is still constant. A query with a leading separator binds ``%`` +
-    escaped(q) + ``%`` and is a full SCAN (a leading wildcard can never use the
-    index — the intentional, documented substring fallback over the modest table).
-    LIKE is case-insensitive for ASCII (matching the NOCASE collation exactly).
+    Match shape (#223): the query binds a SUBSTRING pattern (``%`` + escaped(q)
+    + ``%``) into ``file_path LIKE ? ESCAPE '\\'`` for EVERY query — a bare
+    basename, a mid-path fragment, or a leading-separator path all match. This
+    is a deliberate full SCAN over the modest ``conversation_file_touches``
+    table (a leading wildcard can never use ``idx_file_touches_path``),
+    consistent with how the content facets match; it replaces the former
+    prefix/separator special-case so a basename like ``package.json`` is
+    findable. ``LIKE`` stays case-insensitive (ASCII), matching the NOCASE path
+    index collation even though the index no longer drives a prefix probe.
 
     Filtered-search: the ``filt_sql`` session-scope restriction is a post-match
     ``ft.session_id IN (<subquery>)`` (the same as the message kinds) — there is NO
     dedicated session index (the filter is a small IN over the already-narrowed
     match set, so a ``session_id`` btree earned nothing; dropped per review Minor #2).
     """
-    esc = _like_escape(q)
-    prefix = not q.startswith(_PATH_SEPARATORS)
-    pat = (esc + "%") if prefix else ("%" + esc + "%")
+    pat = _like_pattern(q)
     fpred = _session_filter_pred("ft.session_id", filt_sql)
     fargs = tuple(filt_params)