cctally 1.91.0 → 1.92.1

package/bin/_lib_codex_find_projection.py

@@ -0,0 +1,517 @@
+"""Canonical visible-text projection and matching for Codex conversation find.
+
+The module is deliberately stdlib-only.  Its Markdown scanner models the
+visible text-node boundaries used by the dashboard's ReactMarkdown/remark-gfm
+surface; it is not an HTML renderer and never interprets raw HTML.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+import html
+import re
+from typing import Iterable, Iterator, Sequence
+
+
+CODEX_FIND_PROJECTION_VERSION = 2
+
+
+@dataclass(frozen=True)
+class RenderLeaf:
+    key: str
+    text: str
+
+
+@dataclass(frozen=True)
+class ProjectedLeaf:
+    key: str
+    start: int
+    end: int
+
+
+@dataclass(frozen=True)
+class FindRange:
+    start: int
+    end: int
+
+
+@dataclass(frozen=True)
+class LeafFragment:
+    leaf_key: str
+    start: int
+    end: int
+
+
+class _ProjectionBuilder:
+    def __init__(self) -> None:
+        self.parts: list[str] = []
+        self.leaves: list[dict[str, int | str]] = []
+        self.length = 0
+        self._open_leaf: int | None = None
+
+    def boundary(self) -> None:
+        self._open_leaf = None
+
+    def separator(self, value: str) -> None:
+        if not value:
+            return
+        self.boundary()
+        self.parts.append(value)
+        self.length += len(value)
+
+    def emit(self, value: str, *, boundary: bool = False, key: str | None = None) -> None:
+        if not value:
+            return
+        if boundary:
+            self.boundary()
+        start = self.length
+        self.parts.append(value)
+        self.length += len(value)
+        if key is not None:
+            self.leaves.append({"key": key, "start": start, "end": self.length})
+            self._open_leaf = None
+            return
+        if self._open_leaf is None:
+            self._open_leaf = len(self.leaves)
+            self.leaves.append({
+                "key": f"t{self._open_leaf}",
+                "start": start,
+                "end": self.length,
+            })
+        else:
+            self.leaves[self._open_leaf]["end"] = self.length
+
+    def value(self) -> tuple[str, tuple[ProjectedLeaf, ...]]:
+        return (
+            "".join(self.parts),
+            tuple(ProjectedLeaf(**leaf) for leaf in self.leaves),
+        )
+
+
+_TABLE_DELIMITER_RE = re.compile(
+    r"^\s*\|?\s*:?-{3,}:?\s*(?:\|\s*:?-{3,}:?\s*)+\|?\s*$"
+)
+_BLOCK_PREFIX_RE = re.compile(
+    r"^\s*(?:(?:#{1,6})\s+|>\s?|(?:[-+*]|\d+[.)])\s+)"
+)
+_TASK_MARKER_RE = re.compile(r"^\[[ xX]\]\s+")
+_AUTOLINK_RE = re.compile(r"<((?:https?://|mailto:)[^ <>]+|[^ <>@]+@[^ <>@]+)>")
+
+
+def _table_cells(line: str) -> list[str]:
+    value = line.strip()
+    if value.startswith("|"):
+        value = value[1:]
+    if value.endswith("|"):
+        value = value[:-1]
+    cells: list[str] = []
+    current: list[str] = []
+    escaped = False
+    for char in value:
+        if escaped:
+            current.append(char)
+            escaped = False
+        elif char == "\\":
+            current.append(char)
+            escaped = True
+        elif char == "|":
+            cells.append("".join(current).strip())
+            current = []
+        else:
+            current.append(char)
+    cells.append("".join(current).strip())
+    return cells
+
+
+def _find_closing(source: str, token: str, start: int) -> int:
+    cursor = start
+    while True:
+        found = source.find(token, cursor)
+        if found < 0:
+            return -1
+        backslashes = 0
+        probe = found - 1
+        while probe >= 0 and source[probe] == "\\":
+            backslashes += 1
+            probe -= 1
+        if backslashes % 2 == 0:
+            return found
+        cursor = found + len(token)
+
+
+def _project_inline(source: str, builder: _ProjectionBuilder) -> None:
+    plain: list[str] = []
+
+    def flush() -> None:
+        if plain:
+            builder.emit(html.unescape("".join(plain)))
+            plain.clear()
+
+    cursor = 0
+    while cursor < len(source):
+        if source[cursor] == "\\" and cursor + 1 < len(source):
+            plain.append(source[cursor + 1])
+            cursor += 2
+            continue
+
+        if source[cursor] == "`":
+            run = 1
+            while cursor + run < len(source) and source[cursor + run] == "`":
+                run += 1
+            token = "`" * run
+            close = _find_closing(source, token, cursor + run)
+            if close >= 0:
+                flush()
+                builder.emit(source[cursor + run:close].strip(" "), boundary=True)
+                builder.boundary()
+                cursor = close + run
+                continue
+
+        if source.startswith("![", cursor) or source[cursor] == "[":
+            image = source.startswith("![", cursor)
+            label_start = cursor + (2 if image else 1)
+            label_end = source.find("](", label_start)
+            if label_end >= 0:
+                destination_end = source.find(")", label_end + 2)
+                if destination_end >= 0:
+                    flush()
+                    builder.boundary()
+                    if not image:
+                        _project_inline(source[label_start:label_end], builder)
+                    builder.boundary()
+                    cursor = destination_end + 1
+                    continue
+
+        if source[cursor] == "<":
+            autolink = _AUTOLINK_RE.match(source, cursor)
+            if autolink is not None:
+                flush()
+                builder.boundary()
+                label = autolink.group(1)
+                builder.emit(label[7:] if label.startswith("mailto:") else label)
+                builder.boundary()
+                cursor = autolink.end()
+                continue
+
+        matched_delimiter = False
+        for token in ("**", "__", "~~", "*", "_"):
+            if not source.startswith(token, cursor):
+                continue
+            close = _find_closing(source, token, cursor + len(token))
+            if close < 0 or close == cursor + len(token):
+                continue
+            flush()
+            builder.boundary()
+            _project_inline(source[cursor + len(token):close], builder)
+            builder.boundary()
+            cursor = close + len(token)
+            matched_delimiter = True
+            break
+        if matched_delimiter:
+            continue
+
+        plain.append(source[cursor])
+        cursor += 1
+    flush()
+
+
+def _strip_block_prefix(line: str) -> str:
+    value = _BLOCK_PREFIX_RE.sub("", line, count=1)
+    if _TASK_MARKER_RE.match(value):
+        value = _TASK_MARKER_RE.sub(" ", value, count=1)
+    if value.endswith("  "):
+        value = value[:-2]
+    elif value.endswith("\\"):
+        value = value[:-1]
+    return value
+
+
+def project_markdown(source: str) -> tuple[str, tuple[ProjectedLeaf, ...]]:
+    builder = _ProjectionBuilder()
+    lines = source.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+    blocks: list[tuple[str, object]] = []
+    cursor = 0
+    while cursor < len(lines):
+        line = lines[cursor]
+        if not line.strip():
+            cursor += 1
+            continue
+
+        fence = re.match(r"^\s*(`{3,}|~{3,})(?:[^`]*)$", line)
+        if fence:
+            token = fence.group(1)
+            body: list[str] = []
+            cursor += 1
+            while cursor < len(lines) and not re.match(
+                rf"^\s*{re.escape(token[0])}{{{len(token)},}}\s*$", lines[cursor]
+            ):
+                body.append(lines[cursor])
+                cursor += 1
+            closed = cursor < len(lines)
+            if closed:
+                cursor += 1
+            code = "\n".join(body)
+            if body:
+                code += "\n"
+            blocks.append(("code", code))
+            continue
+
+        if cursor + 1 < len(lines) and "|" in line and _TABLE_DELIMITER_RE.match(lines[cursor + 1]):
+            rows = [_table_cells(line)]
+            cursor += 2
+            while cursor < len(lines) and lines[cursor].strip() and "|" in lines[cursor]:
+                rows.append(_table_cells(lines[cursor]))
+                cursor += 1
+            blocks.append(("table", rows))
+            continue
+
+        paragraph = [_strip_block_prefix(line)]
+        cursor += 1
+        while cursor < len(lines) and lines[cursor].strip():
+            if re.match(r"^\s*(`{3,}|~{3,})", lines[cursor]):
+                break
+            paragraph.append(_strip_block_prefix(lines[cursor]))
+            cursor += 1
+        blocks.append(("paragraph", paragraph))
+
+    for block_index, (kind, value) in enumerate(blocks):
+        if block_index:
+            builder.separator("\n")
+        if kind == "code":
+            builder.emit(str(value), boundary=True)
+            builder.boundary()
+        elif kind == "table":
+            for row_index, row in enumerate(value):
+                if row_index:
+                    builder.separator("\n")
+                for cell_index, cell in enumerate(row):
+                    if cell_index:
+                        builder.separator("\t")
+                    builder.boundary()
+                    _project_inline(cell, builder)
+                    builder.boundary()
+        else:
+            for line_index, line in enumerate(value):
+                if line_index:
+                    builder.separator("\n")
+                _project_inline(line, builder)
+    return builder.value()
+
+
+def project_plain(leaves: Sequence[RenderLeaf]) -> tuple[str, tuple[ProjectedLeaf, ...]]:
+    builder = _ProjectionBuilder()
+    for leaf in leaves:
+        builder.emit(leaf.text, key=leaf.key)
+    return builder.value()
+
+
+_CONTEXT_DIFF_GIT_RE = re.compile(r"diff --git a/\S+ b/\S+")
+_CONTEXT_HUNK_RE = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@")
+_CONTEXT_EXTENDED_HEADER_PREFIXES = (
+    "old mode ", "new mode ", "new file mode ", "deleted file mode ",
+    "rename from ", "rename to ", "copy from ", "copy to ",
+    "similarity index ", "dissimilarity index ", "index ",
+)
+
+
+def _context_is_diff_line(line: str) -> bool:
+    if _CONTEXT_DIFF_GIT_RE.search(line):
+        return True
+    if line.startswith(("--- ", "+++ ", "@@")):
+        return True
+    if line.startswith(_CONTEXT_EXTENDED_HEADER_PREFIXES):
+        return True
+    return line == "" or line[0] in "+- \\"
+
+
+def _segment_context_body(text: str) -> list[tuple[str, str]]:
+    """Mirror ``contextDiff.ts::segmentContextBody`` without rendering HTML."""
+    lines = text.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    segments: list[tuple[str, str]] = []
+    prose: list[str] = []
+    diff: list[str] = []
+    in_diff = False
+
+    def flush(kind: str, values: list[str]) -> None:
+        if values:
+            segments.append((kind, "\n".join(values)))
+            values.clear()
+
+    for line in lines:
+        if not in_diff:
+            match = _CONTEXT_DIFF_GIT_RE.search(line)
+            if match is None:
+                prose.append(line)
+                continue
+            before = line[:match.start()].rstrip()
+            if before:
+                prose.append(before)
+            flush("prose", prose)
+            in_diff = True
+            diff.append(line[match.start():])
+        elif _context_is_diff_line(line):
+            diff.append(line)
+        else:
+            flush("diff", diff)
+            in_diff = False
+            prose.append(line)
+    flush("prose", prose)
+    flush("diff", diff)
+    return segments
+
+
+def _context_diff_rows(text: str) -> list[tuple[int, int, int, str]]:
+    """Mirror the visible row walk in ``contextDiff.ts::parseUnifiedDiff``."""
+    rows: list[tuple[int, int, int, str]] = []
+    file_index = -1
+    hunk_index = -1
+    row_index = 0
+    in_hunk = False
+    for line in text.split("\n"):
+        if _CONTEXT_DIFF_GIT_RE.search(line):
+            file_index += 1
+            hunk_index = -1
+            row_index = 0
+            in_hunk = False
+            continue
+        if _CONTEXT_HUNK_RE.match(line):
+            hunk_index += 1
+            row_index = 0
+            in_hunk = True
+            continue
+        if not in_hunk:
+            continue
+        if line.startswith(("--- ", "+++ ")) or line.startswith(
+            _CONTEXT_EXTENDED_HEADER_PREFIXES
+        ):
+            continue
+        if line == "" or line.startswith("\\"):
+            continue
+        rows.append((file_index, hunk_index, row_index, line[1:]))
+        row_index += 1
+    return rows
+
+
+def _append_projected(
+    builder: _ProjectionBuilder,
+    projected: tuple[str, tuple[ProjectedLeaf, ...]],
+    *,
+    prefix: str,
+) -> None:
+    text, leaves = projected
+    if not text:
+        return
+    start = builder.length
+    builder.parts.append(text)
+    builder.length += len(text)
+    builder.boundary()
+    builder.leaves.extend({
+        "key": f"{prefix}/{leaf.key}",
+        "start": start + leaf.start,
+        "end": start + leaf.end,
+    } for leaf in leaves)
+
+
+def project_context(source: str) -> tuple[str, tuple[ProjectedLeaf, ...]]:
+    """Project visible prose and diff-row leaves from one context body.
+
+    File headers and +/- statistics are derived card chrome, matching #482's
+    rule that only provider-authored render leaves enter the search surface.
+    """
+    builder = _ProjectionBuilder()
+    for segment_index, (kind, text) in enumerate(_segment_context_body(source)):
+        if kind == "prose":
+            projected = project_markdown(text)
+            if not projected[0]:
+                continue
+            if builder.parts:
+                builder.separator("\n")
+            _append_projected(
+                builder, projected, prefix=f"segments.{segment_index}.prose"
+            )
+            continue
+        for file_index, hunk_index, row_index, row_text in _context_diff_rows(text):
+            if not row_text:
+                continue
+            if builder.parts:
+                builder.separator("\n")
+            builder.emit(
+                row_text,
+                key=(
+                    f"segments.{segment_index}.files.{file_index}."
+                    f"hunks.{hunk_index}.rows.{row_index}"
+                ),
+            )
+    return builder.value()
+
+
+def _single_scalar_lower(value: str) -> str:
+    return "".join((lowered if len(lowered := scalar.lower()) == 1 else scalar) for scalar in value)
+
+
+def iter_literal_ranges(
+    text: str, query: str, *, case_sensitive: bool,
+) -> Iterator[FindRange]:
+    if not query:
+        return
+    haystack = text if case_sensitive else _single_scalar_lower(text)
+    needle = query if case_sensitive else _single_scalar_lower(query)
+    if not needle:
+        return
+    cursor = 0
+    while cursor <= len(haystack) - len(needle):
+        found = haystack.find(needle, cursor)
+        if found < 0:
+            break
+        yield FindRange(found, found + len(needle))
+        cursor = found + len(needle)
+
+
+def literal_ranges(text: str, query: str, *, case_sensitive: bool) -> tuple[FindRange, ...]:
+    return tuple(iter_literal_ranges(text, query, case_sensitive=case_sensitive))
+
+
+def iter_regex_ranges(text: str, pattern: re.Pattern[str]) -> Iterator[FindRange]:
+    for match in pattern.finditer(text):
+        if match.end() > match.start():
+            yield FindRange(match.start(), match.end())
+
+
+def regex_ranges(text: str, pattern: re.Pattern[str]) -> tuple[FindRange, ...]:
+    return tuple(iter_regex_ranges(text, pattern))
+
+
+def slice_range_to_leaves(
+    match: FindRange,
+    leaves: Sequence[ProjectedLeaf],
+) -> tuple[LeafFragment, ...]:
+    fragments: list[LeafFragment] = []
+    for leaf in leaves:
+        start = max(match.start, leaf.start)
+        end = min(match.end, leaf.end)
+        if end <= start:
+            continue
+        fragments.append(LeafFragment(
+            leaf_key=leaf.key,
+            start=start - leaf.start,
+            end=end - leaf.start,
+        ))
+    return tuple(fragments)
+
+
+__all__ = [
+    "CODEX_FIND_PROJECTION_VERSION",
+    "FindRange",
+    "LeafFragment",
+    "ProjectedLeaf",
+    "RenderLeaf",
+    "literal_ranges",
+    "iter_literal_ranges",
+    "iter_regex_ranges",
+    "project_markdown",
+    "project_context",
+    "project_plain",
+    "regex_ranges",
+    "slice_range_to_leaves",
+]

package/bin/_lib_codex_harness_preamble.py

@@ -0,0 +1,176 @@
+"""#463 S3 — the closed line reader for Codex tool-output harness preambles.
+
+Pure kernel: no I/O, no DB, no config, and nothing imported beyond ``re``.
+
+Replaces ``_HARNESS_STATUS_RE``, which was one regex written against one assumed
+shape. Executed against every retained tool output on a read-only copy of the
+production store it examined 60,862 outputs, of which 39,942 carry the exact
+``Script completed`` / ``Script failed`` preamble it targets and **0 match**: it
+required a blank line before ``Output:`` and end-of-string after it, while the
+harness writes a single newline and a trailing newline. Five distinct preamble
+grammars exist; that regex targeted one of them.
+
+The replacement is a line reader over a CLOSED vocabulary rather than one regex
+per grammar, deliberately (spec section 4.1). The defect being fixed is a
+pattern written against an assumed shape, and the grammar set was mis-described
+twice while the design was written. A closed vocabulary degrades to ``unknown``
+on an arrangement it has not seen, instead of silently matching nothing.
+
+Two rules bound what may be consumed:
+
+  * **Anchored, never searched.** Matching starts at position zero, because the
+    preamble is positionally guaranteed and a search would match a user's own
+    output somewhere in the middle of a real result.
+  * **Terminated by an ``Output:`` line.** All five observed grammars end that
+    way. Without the terminator a result that legitimately begins with
+    ``Exit code: 0`` would lose its first line; with it, the 10,177 outputs that
+    carry no preamble are untouched by construction.
+
+Any malformed or over-limit field makes the WHOLE preamble unrecognized rather
+than partially parsed, so a near-match degrades to today's behaviour instead of
+stripping a line it did not understand.
+"""
+from __future__ import annotations
+
+import re
+
+# At most this many lines and this many characters are examined before an
+# `Output:` line. Reaching either limit first leaves the text untouched.
+_MAX_PREAMBLE_LINES = 8
+_MAX_PREAMBLE_CHARS = 512
+
+_TERMINATOR = "Output:"
+
+# Value syntaxes (spec section 4.1). A token is 1-64 characters from
+# `[A-Za-z0-9_-]`; an integer is 1-10 digits with an optional leading `-`; a wall
+# time is a decimal number with a unit from a closed set.
+_TOKEN = r"[A-Za-z0-9_-]{1,64}"
+_INT = r"-?\d{1,10}"
+_UNSIGNED_INT = r"\d{1,10}"
+_NUMBER = r"\d+(?:\.\d+)?"
+_UNITS = r"seconds|second|ms"
+
+_CHUNK_ID_RE = re.compile(rf"Chunk ID: ({_TOKEN})")
+_EXIT_CODE_RE = re.compile(rf"Exit code: ({_INT})")
+_WALL_TIME_RE = re.compile(rf"Wall time:? ({_NUMBER}) ({_UNITS})")
+_PROCESS_EXITED_RE = re.compile(rf"Process exited with code ({_INT})")
+_PROCESS_RUNNING_RE = re.compile(rf"Process running with session ID ({_TOKEN})")
+_TOKEN_COUNT_RE = re.compile(rf"Original token count: ({_UNSIGNED_INT})")
+
+_MILLISECOND_UNITS = frozenset({"ms"})
+
+
+class _Reject(Exception):
+    """The line vocabulary refused a line; the whole preamble is unrecognized."""
+
+
+def _wall_time_seconds(number: str, unit: str) -> float:
+    value = float(number)
+    return value / 1000.0 if unit in _MILLISECOND_UNITS else value
+
+
+def _read_line(line: str, observed: dict) -> bool:
+    """Record one recognized preamble line, or raise ``_Reject``.
+
+    Returns True when the line carried a FIELD (so a bare terminator with nothing
+    before it can be refused) and False for a blank separator.
+    """
+    if line == "":
+        # A blank separator carries no field, so tolerating it cannot mis-parse
+        # one, and the terminator plus the two caps still bound what is consumed.
+        # Zero production outputs carry it, but the shipped `session-b-card-wire`
+        # fixture does, and a reader that refused it would silently regress that
+        # fixture's resolved status to `unknown`.
+        return False
+    if line in ("Script completed", "Script failed"):
+        observed["script"] = "completed" if line.endswith("completed") else "failed"
+        return True
+    match = _CHUNK_ID_RE.fullmatch(line)
+    if match is not None:
+        return True                      # deliberately not published (section 4.3)
+    match = _EXIT_CODE_RE.fullmatch(line)
+    if match is not None:
+        observed["exit_code"] = int(match.group(1))
+        return True
+    match = _WALL_TIME_RE.fullmatch(line)
+    if match is not None:
+        observed["wall_time_seconds"] = _wall_time_seconds(
+            match.group(1), match.group(2))
+        return True
+    match = _PROCESS_EXITED_RE.fullmatch(line)
+    if match is not None:
+        observed["exit_code"] = int(match.group(1))
+        return True
+    match = _PROCESS_RUNNING_RE.fullmatch(line)
+    if match is not None:
+        observed["session_announcement"] = match.group(1)
+        observed["running"] = True
+        return True
+    match = _TOKEN_COUNT_RE.fullmatch(line)
+    if match is not None:
+        return True
+    raise _Reject(line)
+
+
+def _resolve_status(observed: dict) -> str:
+    """Spec section 4.2, in priority order.
+
+    ``running`` is explicitly NOT an error: 4,585 outputs are open sessions, and
+    treating them as failures would be worse than the present silence.
+    """
+    if observed.get("script") == "failed":
+        return "failed"
+    if "exit_code" in observed:
+        return "completed" if observed["exit_code"] == 0 else "failed"
+    if observed.get("running"):
+        return "running"
+    if observed.get("script") == "completed":
+        return "completed"
+    return "unknown"
+
+
+def parse_harness_preamble(text: str) -> tuple[dict, str] | None:
+    """``(fields, remainder)`` for a recognized preamble, else ``None``.
+
+    ``fields`` carries ``status`` (one of ``completed``, ``failed``, ``running``,
+    ``unknown``), ``exit_code``, ``wall_time_seconds`` and
+    ``session_announcement``. ``remainder`` is ``text`` with the consumed run,
+    terminator included, removed.
+
+    ``None`` means the caller leaves the text exactly as it found it.
+
+    ``session_announcement`` carries the provider's raw session id for the
+    conversation-level session index ONLY. It is never published on a card:
+    the reader sees a conversation-local ordinal instead (spec section 4.3).
+    """
+    if not isinstance(text, str) or not text:
+        return None
+    observed: dict = {}
+    consumed = 0
+    field_lines = 0
+    for index, raw in enumerate(text.split("\n")):
+        line = raw[:-1] if raw.endswith("\r") else raw
+        consumed += len(raw) + 1
+        if index >= _MAX_PREAMBLE_LINES or consumed > _MAX_PREAMBLE_CHARS:
+            return None
+        if line == _TERMINATOR:
+            # A bare terminator with no field before it is not a preamble — it is
+            # a tool's own first line, and consuming it would delete real output.
+            if field_lines == 0:
+                return None
+            fields = {
+                "status": _resolve_status(observed),
+                "exit_code": observed.get("exit_code"),
+                "wall_time_seconds": observed.get("wall_time_seconds"),
+                "session_announcement": observed.get("session_announcement"),
+            }
+            return fields, text[consumed:]
+        try:
+            if _read_line(line, observed):
+                field_lines += 1
+        except _Reject:
+            return None
+    return None                          # ran out of text before a terminator
+
+
+__all__ = ["parse_harness_preamble"]