cctally 1.40.0 → 1.42.0

package/bin/_lib_conversation.py

@@ -16,6 +16,55 @@ ASSISTANT = "assistant"
 TOOL_RESULT = "tool_result"
 META = "meta"
 
+# Mirror of dashboard/web/src/conversations/systemMarkers.ts::MARKER_RE — anchored
+# whole-string (fullmatch), unrolled-lazy body for linear time (no ReDoS), \1
+# backref forces each close tag to match its open tag. Used to SKIP slash-command
+# plumbing when deriving a conversation title (#165 Q2) AND — at the parser layer
+# (#186) — to classify a slash-command echo carried as a plain user line as META
+# at ingest. MUST stay equivalent to the TS predicate over ASCII whitespace
+# (parity-tested); exotic Unicode/control whitespace is an explicit non-goal.
+# The query kernel re-exports these names for back-compat. See
+# docs/dashboard-gotchas.md.
+_MARKER_TAGS = ("command-name", "command-message", "command-args",
+                "local-command-caveat", "local-command-stdout",
+                "local-command-stderr")
+_MARKER_RE = re.compile(
+    r"\s*(?:<(" + "|".join(_MARKER_TAGS) + r")>(?:(?!</\1>)[\s\S])*</\1>\s*)+"
+)
+
+
+def _is_system_marker(text) -> bool:
+    """True iff `text` is ONLY concatenated command-marker wrappers (slash-command
+    plumbing) — the title-derivation skip predicate AND the parser-layer ingest
+    classifier (#186). `fullmatch` reproduces the TS `^\\s*…\\s*$` anchor (no
+    `$`-before-trailing-`\\n` foot-gun)."""
+    return bool(text) and _MARKER_RE.fullmatch(text) is not None
+
+
+# #186: strip ANSI terminal control sequences from captured prose/thinking so a
+# slash-command stdout echo (which keeps its SGR styling, e.g. `\x1b[1mFable
+# 5\x1b[22m`) never leaks literal `^[[1m` control codes into a title, an outline
+# label, or body text — and so FTS indexes clean tokens (`Fable`, not
+# `1mFable`). Three alternatives: CSI (covers all SGR `…m`), OSC (BEL- or
+# ST-terminated, or unterminated), and a lone/truncated ESC. Ordinary `[`,
+# digits, `m` in prose are untouched — they match only as part of a real
+# `\x1b[`-led sequence. Conceptually mirrors the client `parseAnsi` cleanup.
+# Deliberately NOT applied in `_stringify` / tool_result text: `AnsiText` (#177
+# S3) renders Bash stdout/stderr SGR colors, so that path keeps raw ANSI.
+_ANSI_RE = re.compile(
+    r"\x1b\[[0-9;:?]*[ -/]*[@-~]"           # CSI (covers all SGR …m)
+    r"|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)?"  # OSC (BEL/ST terminated or unterminated)
+    r"|\x1b"                                # lone/truncated ESC
+)
+
+
+def _strip_ansi(text):
+    """Remove ANSI control sequences (terminal SGR/CSI/OSC). Ordinary '[',
+    digits, 'm' in prose are untouched — they match only inside a real \\x1b[
+    run. Empty / None passes through unchanged. (#186)"""
+    return _ANSI_RE.sub("", text) if text else text
+
+
 _TOOL_RESULT_CAP = 16000   # was 4000; full text always re-derivable from JSONL
 _INPUT_LEAF_CAP = 8000     # max chars per string leaf in a bounded tool input
 _INPUT_TOTAL_CAP = 32000   # honesty backstop on the serialized bounded input
@@ -53,7 +102,13 @@ class MessageRow:
     stop_reason: "str | None" = None
     attribution_skill: "str | None" = None
     attribution_plugin: "str | None" = None
-    search_aux: str = ""
+    # #177 S6: split non-prose search columns, derived by _derive_search_columns
+    # on the FINAL post-augment blocks (see _normalize). ``search_aux`` is kept
+    # physically (legacy column) but NEVER assigned a non-empty value — the
+    # consolidated multi-column FTS reads search_tool/search_thinking instead.
+    search_tool: str = ""
+    search_thinking: str = ""
+    search_aux: str = ""   # documented-dead (#177 S6); always "" on new rows
 
 
 def iter_message_rows(fh, path_str):
@@ -107,7 +162,7 @@ def _normalize(obj, t, offset):
     msg = obj.get("message")
     if not isinstance(msg, dict):
         msg = {}
-    blocks, text, aux = _blocks_and_text(msg.get("content"))
+    blocks, text = _blocks_and_text(msg.get("content"))
     if t == "assistant":
         entry_type = ASSISTANT
     elif any(b["kind"] == "tool_result" for b in blocks):
@@ -137,9 +192,30 @@ def _normalize(obj, t, offset):
         # tool_result block still folds as a result.
         entry_type = META
         text = ""
+    elif blocks and all(b["kind"] == "text" for b in blocks) and _is_system_marker(text):
+        # A slash-command echo carried as a plain user line (NOT isMeta): the
+        # user did not type it. `<local-command-stdout>…</local-command-stdout>`
+        # & friends arrive as ordinary user content, so without this branch they
+        # render as a "YOU" prompt and poison the conversation title (#186).
+        # Classify META + text="" exactly like the isMeta branch above so the
+        # body stays out of FTS, title derivation, and the entry_type='human'
+        # prompts facet; the body survives in blocks_json for rendering as a
+        # "System marker" pill. The all-text guard mirrors the tool_result
+        # block-shape gate (Codex P1b) so an attachment-bearing row — a marker
+        # text block PLUS an image — is never folded.
+        entry_type = META
+        text = ""
     else:
         entry_type = HUMAN
     is_asst = t == "assistant"
+    # #177 S6: derive the split search columns on the FINAL post-augment blocks
+    # (every _attach_* pass above has already merged bash_stderr / answers /
+    # annotations into the blocks). This is the SAME chokepoint the migration-010
+    # backfill runs on json.loads(blocks_json), so live ingest and backfill
+    # produce byte-identical values (the parity invariant). Computed immediately
+    # before json.dumps(blocks) so blocks_json and the columns agree by
+    # construction.
+    search_tool, search_thinking = _derive_search_columns(blocks)
     return MessageRow(
         byte_offset=offset,
         session_id=obj.get("sessionId"),
@@ -163,26 +239,30 @@ def _normalize(obj, t, offset):
         stop_reason=msg.get("stop_reason") if is_asst else None,
         attribution_skill=obj.get("attributionSkill"),
         attribution_plugin=obj.get("attributionPlugin"),
-        search_aux=aux,
+        search_tool=search_tool,
+        search_thinking=search_thinking,
+        # search_aux stays "" (documented-dead, #177 S6) — the consolidated
+        # multi-column FTS reads search_tool/search_thinking instead.
     )
 
 
 def _blocks_and_text(content):
-    """Return (normalized blocks list, indexed-prose string, search_aux string).
+    """Return (normalized blocks list, indexed-prose string).
 
     Prose (``text``) = joined ``text`` blocks only (thinking / tool_use /
-    tool_result excluded — those go to the prose FTS via the ``text`` column).
-    ``search_aux`` (#177) = the non-prose searchable content: bounded tool-input
-    string leaves, the (capped) tool_result ``text``, and the thinking text
-    capped at ``_TOOL_RESULT_CAP`` (code-review I2 — the FULL thinking still
-    lives in ``blocks_json`` for rendering; only this aux index entry is capped
-    so the second FTS index doesn't double the at-rest cost of large thinking) —
-    indexed by the parallel ``conversation_fts_aux`` so tool content stays
-    searchable without polluting prose FTS. Prose is deliberately excluded from
-    aux (it is already in ``text``)."""
+    tool_result excluded — those go to the split search columns via
+    ``_derive_search_columns``, which runs in ``_normalize`` on the FINAL
+    post-augment blocks). #177 S6 dropped the in-loop ``search_aux`` accumulation
+    that used to be the third return element: deriving the search columns here
+    (pre-augment) would miss bash_stderr / answers / annotations and break the
+    ingest==backfill parity invariant, so the derivation moved to the
+    post-augment chokepoint."""
     if isinstance(content, str):
-        return (([{"kind": "text", "text": content}] if content else []), content, "")
-    blocks, texts, aux_parts = [], [], []
+        # #186: strip ANSI on the str-content path (the slash-command-stdout
+        # carrier) so neither the indexed prose nor the rendered block leaks SGR.
+        content = _strip_ansi(content)
+        return (([{"kind": "text", "text": content}] if content else []), content)
+    blocks, texts = [], []
     if isinstance(content, list):
         # #177 S4: ordinal among media items at THIS list level, keyed by the
         # object identity of each image/document item, so the placeholder writers
@@ -193,13 +273,12 @@ def _blocks_and_text(content):
                 continue
             bt = b.get("type")
             if bt == "text":
-                txt = b.get("text", "") or ""
+                txt = _strip_ansi(b.get("text", "") or "")   # #186: strip SGR
                 blocks.append({"kind": "text", "text": txt})
                 texts.append(txt)
             elif bt == "thinking":
-                think = b.get("thinking", "") or ""
+                think = _strip_ansi(b.get("thinking", "") or "")  # #186: strip SGR
                 blocks.append({"kind": "thinking", "text": think})  # FULL text for render
-                aux_parts.append(think[:_TOOL_RESULT_CAP])          # aux index capped (I2)
             elif bt == "tool_use":
                 bounded, input_trunc = _bound_input(b.get("input"))
                 block = {"kind": "tool_use", "name": b.get("name"),
@@ -211,7 +290,6 @@ def _blocks_and_text(content):
                 st = inp.get("subagent_type") if isinstance(inp, dict) else None
                 if isinstance(st, str) and st:        # #166: spawn kind (Agent/Task)
                     block["subagent_type"] = st
-                aux_parts.extend(_aux_strings(bounded))
                 blocks.append(block)
             elif bt == "tool_result":
                 raw = _stringify(b.get("content"))
@@ -230,14 +308,12 @@ def _blocks_and_text(content):
                 if media:                      # omitted when empty (additive)
                     block["media"] = media
                 blocks.append(block)
-                aux_parts.append(clipped)
             elif bt in _MEDIA_BLOCK_TYPES:
                 blocks.append({"kind": bt, **_media(b.get("source")),
                                "index": media_index[id(b)]})
             elif bt == "tool_reference":
                 blocks.append({"kind": "tool_reference", "name": b.get("name")})
-    return (blocks, "\n".join(t for t in texts if t),
-            "\n".join(a for a in aux_parts if a))
+    return (blocks, "\n".join(t for t in texts if t))
 
 
 _SUBAGENT_META_KEYS = (
@@ -558,8 +634,51 @@ def _bound_input(inp):
     return (bounded, state["truncated"])
 
 
+def _derive_search_columns(blocks):
+    """(search_tool, search_thinking) from the FINAL normalized blocks list.
+
+    MUST run post-augmentation (bash_stderr / answers / annotations already
+    merged into the blocks) so live ingest and the migration-010 backfill from
+    blocks_json produce byte-identical values (#177 S6 parity invariant — the
+    chokepoint runs in _normalize right before json.dumps(blocks), and the
+    backfill runs it on json.loads(blocks_json)). Caps are PER-BLOCK
+    (_TOOL_RESULT_CAP each), matching the old search_aux semantics — NOT a
+    whole-column total. Prose is excluded (already in the ``text`` column).
+
+    ``search_tool`` = bounded tool-input string leaves + clipped tool_result
+    ``text`` + ``bash_stderr`` + bounded AskUserQuestion answers/annotations
+    (real ingest stamps these as ``ask_answers``/``ask_annotations``; the raw
+    ``answers``/``annotations`` keys are also read for forward compatibility and
+    the synthetic-block unit tests). ``search_thinking`` = thinking text."""
+    tool_parts, think_parts = [], []
+    for b in blocks if isinstance(blocks, list) else []:
+        if not isinstance(b, dict):
+            continue
+        k = b.get("kind")
+        if k == "thinking":
+            t = b.get("text") or ""
+            if t:
+                think_parts.append(t[:_TOOL_RESULT_CAP])
+        elif k == "tool_use":
+            tool_parts.extend(
+                s[:_TOOL_RESULT_CAP] for s in _aux_strings(b.get("input")))
+        elif k == "tool_result":
+            t = b.get("text") or ""
+            if t:
+                tool_parts.append(t[:_TOOL_RESULT_CAP])
+            stderr = b.get("bash_stderr") or ""
+            if stderr:
+                tool_parts.append(stderr[:_TOOL_RESULT_CAP])
+            # answers/annotations: real ingest uses the ``ask_``-prefixed keys;
+            # the bare keys are read too (forward-compat + synthetic unit tests).
+            for key in ("ask_answers", "ask_annotations", "answers", "annotations"):
+                tool_parts.extend(
+                    s[:_TOOL_RESULT_CAP] for s in _aux_strings(b.get(key)))
+    return ("\n".join(tool_parts), "\n".join(think_parts))
+
+
 def _aux_strings(v):
-    """Yield string leaves from a bounded input value (for the search_aux blob)."""
+    """Yield string leaves from a bounded input value (for the search columns)."""
     if isinstance(v, str):
         if v:
             yield v