cctally 1.90.1 → 1.92.0

package/bin/_lib_codex_js_scan.py

@@ -0,0 +1,254 @@
+"""#463 S3 — a bounded lexical scanner for `tools.<name>(` in Codex programs.
+
+Pure kernel: no I/O, nothing imported beyond ``re``.
+
+Why this is a tokenizer and not a search. 17,777 uncarded `exec` calls, 51% of
+the uncarded set, are JavaScript programs that declare constants, filter
+`ALL_TOOLS`, await `Promise.all` and invoke `tools.exec_command`,
+`tools.write_stdin` or other tools. The obvious way to card them is a textual
+scan for `tools.<name>(`, and it is not acceptable:
+`tests/test_codex_conversation_normalization.py` asserts that a call written
+inside a string literal, inside a `//` comment and inside a regex literal each
+decode to `None`, and a textual scanner would decode all three — fabricating
+tool activity from a comment. A `complete: false` flag communicates omission; it
+cannot make a false positive truthful.
+
+So the source is walked ONCE through a small state machine that skips string
+literals, template literals (including `${}` substitutions, which return to code
+state), regex literals and both comment forms, and an invocation is recognized
+only at a genuine token position whose member path is exactly `tools.<name>`.
+
+It stays closed, bounded and non-executing. Lexing is not evaluation: nothing
+here runs, resolves an identifier or follows a reference, so an aliased
+`alias.exec_command(…)` and a computed `tools[key](…)` are both invisible by
+construction, which is correct — they are not provably `tools.<name>`.
+
+Anything that cannot be lexed cleanly returns ``None``, and the caller then
+produces no card at all rather than a partial one.
+"""
+from __future__ import annotations
+
+import re
+
+# The same ceiling `decode_tool_call_card` already applies to a harness body.
+# Duplicated as a literal rather than imported, because importing the decoder
+# module here would be circular — that module imports this one.
+_PARSE_CAP = 1_000_000
+
+_IDENT_START = re.compile(r"[A-Za-z_$]")
+_IDENT_RE = re.compile(r"[A-Za-z_$][A-Za-z0-9_$]*")
+
+# Anchored at a genuine `tools` token. Whitespace is tolerated around the dot and
+# before the argument list because real programs wrap long chains across lines.
+_MEMBER_CALL_RE = re.compile(
+    r"tools\s*\.\s*([A-Za-z_$][A-Za-z0-9_$]*)\s*\(")
+
+# A `/` opens a regex literal only when the previous significant token is one of
+# these, or when it is the first token in the source. After a value — an
+# identifier, a number, a string, `)` or `]` — a `/` is division. Getting this
+# wrong in the permissive direction would swallow the rest of the program into a
+# regex and silently lose every invocation after it.
+_REGEX_ALLOWED_PUNCTUATION = frozenset("(,=:[!&|?{};+-*%<>~^")
+_REGEX_ALLOWED_KEYWORDS = frozenset({
+    "return", "typeof", "instanceof", "in", "of", "new", "delete", "void",
+    "case", "do", "else", "yield", "await", "throw",
+})
+
+# Sentinel `prev` value meaning "a completed value", after which `/` is division.
+_VALUE = "\0value"
+
+_DEFAULT = 0
+_SINGLE = 1
+_DOUBLE = 2
+_TEMPLATE = 3
+_LINE_COMMENT = 4
+_BLOCK_COMMENT = 5
+_REGEX = 6
+
+
+def _regex_allowed(prev: str | None) -> bool:
+    if prev is None:
+        return True
+    if prev == _VALUE:
+        return False
+    if len(prev) == 1:
+        return prev in _REGEX_ALLOWED_PUNCTUATION
+    return prev in _REGEX_ALLOWED_KEYWORDS
+
+
+def scan_tool_invocations(
+    source: str, *, limit: int,
+) -> list[tuple[str, int]] | None:
+    """``[(tool_name, arg_open_paren_index), …]``, or ``None``.
+
+    ``None`` means the source could not be lexed cleanly — an unterminated
+    string, template, regex or block comment, or a source past the parse cap —
+    and the caller must then produce no card.
+
+    An empty list means the source lexed but contains no `tools.<name>(` at a
+    genuine token position. A call inside a literal or a comment is therefore
+    indistinguishable from no call at all, which is the point.
+
+    ``limit`` bounds how many invocations are COLLECTED, not how far the scan
+    runs: the walk always continues to the end of the source, because an
+    unterminated literal after the limit still means the source could not be
+    lexed and no card may be built from it.
+    """
+    if not isinstance(source, str) or len(source) > _PARSE_CAP:
+        return None
+    found: list[tuple[str, int]] = []
+    state = _DEFAULT
+    prev: str | None = None
+    brace_depth = 0
+    template_returns: list[int] = []
+    in_char_class = False
+    index = 0
+    size = len(source)
+    while index < size:
+        char = source[index]
+        if state == _DEFAULT:
+            if char in " \t\r\n":
+                index += 1
+                continue
+            if char == "/" and index + 1 < size and source[index + 1] == "/":
+                state = _LINE_COMMENT
+                index += 2
+                continue
+            if char == "/" and index + 1 < size and source[index + 1] == "*":
+                state = _BLOCK_COMMENT
+                index += 2
+                continue
+            if char == "/":
+                if _regex_allowed(prev):
+                    state = _REGEX
+                    in_char_class = False
+                    index += 1
+                    continue
+                prev = "/"
+                index += 1
+                continue
+            if char == "'":
+                state = _SINGLE
+                index += 1
+                continue
+            if char == '"':
+                state = _DOUBLE
+                index += 1
+                continue
+            if char == "`":
+                state = _TEMPLATE
+                index += 1
+                continue
+            if char == "{":
+                brace_depth += 1
+                prev = "{"
+                index += 1
+                continue
+            if char == "}":
+                if template_returns and brace_depth == template_returns[-1]:
+                    template_returns.pop()
+                    state = _TEMPLATE
+                    prev = _VALUE
+                    index += 1
+                    continue
+                brace_depth -= 1
+                prev = "}"
+                index += 1
+                continue
+            if _IDENT_START.match(char):
+                # A `tools` token is only a member path when nothing binds to it
+                # on the left. The test is the previous significant TOKEN, not
+                # the character immediately before `tools`: JavaScript permits
+                # whitespace and newlines around `.`, so `evil . tools.x` and
+                # `evil.\n  tools.x` are member accesses on `evil` even though
+                # the character before `tools` is a space. `prev` is unchanged by
+                # the whitespace branch above, so it still holds the `.`.
+                if prev != ".":
+                    match = _MEMBER_CALL_RE.match(source, index)
+                    if match is not None:
+                        if len(found) < limit:
+                            found.append((match.group(1), match.end() - 1))
+                        prev = "("
+                        index = match.end()
+                        continue
+                word = _IDENT_RE.match(source, index)
+                prev = word.group(0)
+                index = word.end()
+                continue
+            if char.isdigit():
+                index += 1
+                prev = _VALUE
+                continue
+            prev = char
+            index += 1
+            continue
+        if state in (_SINGLE, _DOUBLE):
+            if char == "\\":
+                index += 2
+                continue
+            if (state == _SINGLE and char == "'") or (state == _DOUBLE and char == '"'):
+                state = _DEFAULT
+                prev = _VALUE
+                index += 1
+                continue
+            if char == "\n":
+                return None              # an unterminated ordinary string literal
+            index += 1
+            continue
+        if state == _TEMPLATE:
+            if char == "\\":
+                index += 2
+                continue
+            if char == "`":
+                state = _DEFAULT
+                prev = _VALUE
+                index += 1
+                continue
+            if char == "$" and index + 1 < size and source[index + 1] == "{":
+                template_returns.append(brace_depth)
+                state = _DEFAULT
+                prev = "{"
+                index += 2
+                continue
+            index += 1
+            continue
+        if state == _LINE_COMMENT:
+            if char == "\n":
+                state = _DEFAULT
+            index += 1
+            continue
+        if state == _BLOCK_COMMENT:
+            if char == "*" and index + 1 < size and source[index + 1] == "/":
+                state = _DEFAULT
+                index += 2
+                continue
+            index += 1
+            continue
+        # _REGEX
+        if char == "\\":
+            index += 2
+            continue
+        if char == "[":
+            in_char_class = True
+            index += 1
+            continue
+        if char == "]":
+            in_char_class = False
+            index += 1
+            continue
+        if char == "/" and not in_char_class:
+            state = _DEFAULT
+            prev = _VALUE
+            index += 1
+            continue
+        if char == "\n":
+            return None                  # an unterminated regex literal
+        index += 1
+    if state != _DEFAULT and state != _LINE_COMMENT:
+        return None                      # unterminated literal or block comment
+    if template_returns:
+        return None                      # unterminated `${` substitution
+    return found
+
+
+__all__ = ["scan_tool_invocations"]

package/bin/_lib_codex_landmarks.py

@@ -0,0 +1,309 @@
+"""Read-time landmark derivation for the Codex conversation outline (#463 S4).
+
+Pure kernel: no SQLite, no I/O, and nothing from the query layer. It consumes
+the S3 card decoders in ``_lib_codex_conversation`` and the S2 heading
+decomposition in ``_lib_codex_reasoning_headings``, and returns small derived
+facts — a failure verdict per outcome-bearing row, the authored reasoning
+headings of a reasoning row, and the per-file touches of a patch event.
+
+**Everything here is read-time only.** Nothing it produces may reach
+``codex_conversation_messages.detail_json``: those bytes feed ``_row_source_bytes``,
+which drives ``PAGE_SOURCE_BYTE_BUDGET`` page boundaries, so persisting a derived
+field would make two conversations with identical content paginate differently
+according to which binary ingested them. The standing guard is
+``tests/test_codex_conversation_normalization.py::test_s3_no_read_time_enrichment_is_ever_persisted``.
+"""
+from __future__ import annotations
+
+import dataclasses
+import re
+from typing import Any, Iterable
+
+from _lib_codex_reasoning_headings import decompose_reasoning_headings
+
+# A physical row address — ``(source_path, line_offset)``, the key every
+# conversation table is unique on and the one the payload pass joins by.
+Position = tuple[str, int]
+
+# The two status strings that mean the call failed. `decode_tool_output_card`
+# already sets `is_error` from exactly this set, and `classify_tool_failure`
+# below applies it to the patch and completion cards too so one definition
+# covers every family.
+#
+# `"error"` is deliberately in this set. The CLIENT's `OUTCOME_STATUSES`
+# excludes it, so a card carrying `status: "error"` on a call whose call-side
+# card is not `terminal` collapses to `unknown` and is not flagged there. Spec
+# §6.4 takes correctness over bug-compatibility: the server states one
+# enumerated classification and Task 9 brings the client to it, because
+# asserting "the server matches the client exactly" would have frozen a defect.
+#
+# `"running"` and `"unknown"` are NOT failures. `unknown` is a real state
+# covering 17.6% of outputs — 4,585 of them are open sessions, measured — rather
+# than an absence, and reporting it as a failure would invent errors that the
+# reader could then not find.
+_FAILED_STATUSES = frozenset({"failed", "error"})
+
+
+def classify_tool_failure(cards: Any) -> bool:
+    """Did this tool call fail? One enumerated definition, spec §6.4.
+
+    ``cards`` maps a card family to the card the decoders produced for this
+    call — ``terminal_output``, ``patch``, and ``web``/``mcp`` whose value holds
+    the folded ``completion``. An absent family contributes nothing; an
+    unrecognized one is ignored rather than guessed at.
+
+    Each disjunct is named with the card it comes from:
+    """
+    if not isinstance(cards, dict):
+        return False
+
+    # 1. The result side of a terminal call — `decode_tool_output_card`, whose
+    #    own `is_error` is the resolved five-grammar verdict.
+    output = cards.get("terminal_output")
+    if isinstance(output, dict):
+        if output.get("is_error") is True:
+            return True
+        # 2. The same card's resolved status, which covers a card built by a
+        #    caller that did not carry `is_error` forward.
+        if output.get("status") in _FAILED_STATUSES:
+            return True
+
+    # 3. A patch, from either side — `decode_patch_event_card`'s `success`, and
+    #    the standalone patch event's own `status`, which the client treats as
+    #    an error independently of `success`.
+    patch = cards.get("patch")
+    if isinstance(patch, dict):
+        if patch.get("success") is False:
+            return True
+        if patch.get("status") in _FAILED_STATUSES:
+            return True
+
+    # 4/5. The web-search and MCP completions — `decode_secondary_event_card`,
+    #      reached through the call-side card's folded `completion`.
+    for family in ("web", "mcp"):
+        holder = cards.get(family)
+        if not isinstance(holder, dict):
+            continue
+        completion = holder.get("completion")
+        if isinstance(completion, dict) and completion.get("status") in _FAILED_STATUSES:
+            return True
+
+    return False
+
+
+def reasoning_heading_texts(payload: Any) -> list[str] | None:
+    """The authored headings of one reasoning payload's ``summary``, or ``None``.
+
+    Lifted verbatim out of ``_lib_codex_conversation_query._reasoning_headings``
+    so the outline and the reader decompose by ONE rule. S2's §4.6 precedent is
+    that the wire publishes every heading and the render layer dedupes; a second
+    copy of this parse is exactly how the two would drift.
+
+    Headings come from ``payload["summary"]`` ONLY. ``payload["content"]`` is the
+    body, which stays disclosure content and is never decomposed.
+
+    All-or-nothing. When the payload is absent, unreadable or malformed this
+    returns ``None`` and the caller omits the field entirely, so the client falls
+    back to the stored ``title``/``summary`` rendering. Decomposition never fails
+    the request and never partially populates.
+    """
+    if not isinstance(payload, dict):
+        return None
+    summary = payload.get("summary")
+    if not isinstance(summary, list) or not summary:
+        return None
+    entries: list[str] = []
+    for entry in summary:
+        if not isinstance(entry, dict):
+            return None
+        text = entry.get("text")
+        # Mirror `_join_content_texts`, which is what produced the stored
+        # summary: it keeps non-empty string `text` leaves and ignores the rest.
+        if text is None:
+            continue
+        if not isinstance(text, str):
+            return None
+        if text:
+            entries.append(text)
+    headings = decompose_reasoning_headings(entries)
+    return headings or None
+
+
+# `@@ -<old start>[,<old count>] +<new start>[,<new count>] @@`. An omitted count
+# means 1, which is what a single-line hunk emits.
+_HUNK_HEADER_RE = re.compile(r"^@@ -\d+(?:,(\d+))? \+\d+(?:,(\d+))? @@")
+
+
+def _diff_counts(diff: str) -> tuple[int, int] | None:
+    """Added and removed line counts of one unified diff, or ``None``.
+
+    Counted INSIDE the ``@@`` hunks, never by prefix over the whole text. A
+    prefix filter cannot separate a file header from content that looks like
+    one: removing the SQL comment ``-- legacy`` renders ``--- legacy`` and
+    adding ``++i;`` renders ``+++i;``, so excluding every line that starts with
+    ``---`` or ``+++`` drops real changed lines and silently understates the
+    diff — the same undercount §4.5 exists to prevent, reached by a different
+    route. A header can only appear before a hunk opens, and the hunk header
+    states how many old-side and new-side lines follow, so inside a hunk nothing
+    has to be recognised by its prefix alone.
+
+    ``None`` when the text carries no hunk at all. Such a string is not a
+    unified diff, and counting its ``+``/``-`` prefixed lines would be the same
+    guess; the caller reports an undetermined count rather than a wrong one.
+    """
+    added = removed = 0
+    old_left = new_left = 0
+    in_hunk = saw_hunk = False
+    for line in diff.split("\n"):
+        if not in_hunk:
+            header = _HUNK_HEADER_RE.match(line)
+            if header is None:
+                continue
+            saw_hunk = True
+            old_left = int(header.group(1)) if header.group(1) is not None else 1
+            new_left = int(header.group(2)) if header.group(2) is not None else 1
+            in_hunk = old_left > 0 or new_left > 0
+            continue
+        if line.startswith("\\"):
+            # "" belongs to neither side's line count.
+            continue
+        if line.startswith("+"):
+            added += 1
+            new_left -= 1
+        elif line.startswith("-"):
+            removed += 1
+            old_left -= 1
+        else:
+            old_left -= 1
+            new_left -= 1
+        if old_left <= 0 and new_left <= 0:
+            in_hunk = False
+    return (added, removed) if saw_hunk else None
+
+
+def patch_file_touches(payload: Any) -> list[dict]:
+    """Per-file touches of one ``patch_apply_end`` payload, in provider order.
+
+    Each entry is ``{"path", "op", "added", "removed"}``; ``added``/``removed``
+    are ``None`` when the count cannot be determined, never 0, because 0 is a
+    claim and an undetermined count is not.
+
+    **Counted from the UNBOUNDED raw ``changes`` entry, before card allocation**
+    (spec §4.5). ``decode_patch_event_card`` shares one 16,000-character budget
+    across stdout, stderr and every file and caps the file list at 128, and S3
+    measured 85 real change entries over that budget — so counting `+`/`-` lines
+    out of a served ``unified_diff`` silently undercounts, and a wholly clipped
+    diff would read as no change at all despite complete evidence sitting in the
+    payload. Presentation cards and whole-session statistics have different
+    bounds and must not share one.
+
+    Only the DICT-shaped ``changes`` is a real production source. S3 measured
+    all 4,829 patch events arriving as a dict keyed by file path; #489 taught
+    the stored file-search projection to consume that shape too. The legacy
+    list shape stays supported here and at ingest so a provider change cannot
+    silently produce an empty file list.
+    """
+    if not isinstance(payload, dict) or payload.get("type") != "patch_apply_end":
+        return []
+    changes = payload.get("changes")
+    touches: list[dict] = []
+    if isinstance(changes, dict):
+        items: Iterable[tuple[Any, Any]] = changes.items()
+    elif isinstance(changes, list):
+        # The path lives on the entry in this shape, and the kind is `status`
+        # rather than `type` — the two shapes disagree on both.
+        items = (
+            (change.get("path"), change) for change in changes
+            if isinstance(change, dict)
+        )
+    else:
+        return []
+    for path, change in items:
+        if not isinstance(path, str) or not isinstance(change, dict):
+            continue
+        kind = change.get("type")
+        if not isinstance(kind, str):
+            kind = change.get("status")
+        added = removed = None
+        diff = change.get("unified_diff")
+        if isinstance(diff, str):
+            counted = _diff_counts(diff)
+            if counted is not None:
+                added, removed = counted
+        elif kind in {"add", "delete"} and isinstance(change.get("content"), str):
+            lines = change["content"].split("\n")
+            if lines and lines[-1] == "":
+                lines.pop()
+            added, removed = ((len(lines), 0) if kind == "add"
+                              else (0, len(lines)))
+        touches.append({
+            "path": path,
+            "op": kind if isinstance(kind, str) else None,
+            "added": added,
+            "removed": removed,
+        })
+    return touches
+
+
+def fold_owner_by_position(
+    groups: Iterable[Iterable[Position]],
+) -> dict[Position, Position]:
+    """Map every row position to the position of its fold group's first row.
+
+    That first row is the ``tool_call`` a failing ``tool_output`` or completion
+    event belongs to. ``_fold_groups_for_item`` computes this membership
+    payload-free and ``_build_segment_index`` discarded it, so the outline had no
+    way to say WHICH call a failure belongs to — only that the segment contained
+    one.
+
+    Grouping is a conservative SUPERSET of what the block builder actually folds,
+    which is the safe direction here: attributing a failure to the call it was
+    grouped with can at worst name a call that did not fold, and never splits a
+    call from an outcome that did.
+    """
+    owners: dict[Position, Position] = {}
+    for group in groups:
+        head: Position | None = None
+        for position in group:
+            if head is None:
+                head = position
+            owners[position] = head
+    return owners
+
+
+@dataclasses.dataclass
+class EventDerivation:
+    """Everything the outline derives from one scoped pass over the raw payloads.
+
+    Keyed by physical position throughout, because that is the only identity the
+    message table and the event table share, and because the item and segment
+    keys a landmark ultimately carries are minted later from the rows these
+    positions name.
+
+    **Mutable, and filled in by the pass as it streams.** The maps are complete
+    only once ``_derive_outline_events`` has returned; a partly-filled instance
+    is never handed to a consumer. The dataclass is deliberately not frozen:
+    freezing rebinding of the three attributes while the dicts they name are
+    mutated in place would advertise an immutability this object does not have.
+
+    ``errors_by_position`` answers "did this call fail" for the outcome-bearing
+    rows the pass classified — a ``tool_output``, or a ``patch_apply_end`` /
+    ``web_search_end`` / ``mcp_tool_call_end`` event. **Absence is a third state,
+    not ``False``.** A position is absent when the scope did not select it, and
+    equally when it was selected and could not be classified — its event row is
+    gone, its payload does not parse, or the decoder declined the shape. The
+    caller cannot tell those apart from this map and must not read absence as a
+    claim that the call succeeded: a count derived from it is a count of
+    failures FOUND, and a route that needs to distinguish "looked and found
+    none" from "could not look" compares these keys against the in-scope set it
+    supplied.
+    """
+
+    errors_by_position: dict[Position, bool] = dataclasses.field(default_factory=dict)
+    patch_files_by_position: dict[Position, list[dict]] = dataclasses.field(
+        default_factory=dict)
+    headings_by_position: dict[Position, list[str]] = dataclasses.field(
+        default_factory=dict)
+
+    def failing_positions(self) -> set[Position]:
+        return {pos for pos, failed in self.errors_by_position.items() if failed}

package/bin/_lib_codex_reasoning_headings.py

@@ -0,0 +1,73 @@
+"""Pure decomposition of a Codex reasoning aggregate into its authored headings.
+
+#463 S2 §2.4. Codex writes reasoning as short bold headings. When an aggregate
+holds several of them joined by newlines, ``_REASONING_TITLE_RE`` cannot
+fullmatch the whole summary and the blob becomes one ``summary``, which the
+reader renders as one clipped line. Measured over a real store: 5,081 of the
+5,234 summary-only reasoning blocks are entirely bold-heading lines, holding
+12,323 individual headings between them, and every one of those headings is
+today invisible past the first thirty characters of the first one.
+
+This module recovers the individual headings WITHOUT touching the stored
+projection. That seam is load-bearing and is the trap most likely to be walked
+into again: ``_row_is_reasoning_title`` reads the stored reasoning projection to
+decide whether a row is a title boundary, and that is one of segmentation's two
+semantic boundaries. Decomposing inside ``_reasoning_projection`` would change
+which rows are boundaries and therefore move segment boundaries, which #463 S1's
+contract forbids. Decomposition therefore runs at READ time, over the retained
+payload's ``summary`` entries, and the stored ``title``, ``summary`` and ``body``
+keep exactly today's values.
+
+No I/O, no database, no imports beyond ``re``.
+"""
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable
+
+# The same shape ``_lib_codex_conversation._REASONING_TITLE_RE`` recognises,
+# applied per LINE rather than to the whole summary. Deliberately NOT carrying
+# the stored projection's additional guard (which rejects a title containing
+# ``**``): the projection is a frozen segmentation input, this is a read-time
+# rendering concern, and §2.4 states the per-line rule without that guard.
+_HEADING_LINE_RE = re.compile(r"\A\*\*([^\n]+)\*\*\Z")
+
+# Codex interleaves this literal between headings in a minority of aggregates
+# (123 of 5,234 measured). It is layout, not content, so a decomposed entry
+# discards it.
+_SEPARATOR_LINE = "<!-- -->"
+
+
+def decompose_reasoning_headings(entries: Iterable[object]) -> list[str]:
+    """Heading texts for one reasoning aggregate, in entry order.
+
+    ``entries`` is the ordered ``text`` values of the retained payload's
+    ``summary`` entries. The unit of decision is the ENTRY and the decision is
+    all-or-nothing: an entry yields several headings only when, after discarding
+    separator lines, every remaining line is a heading line and at least one
+    remains. Otherwise it yields exactly one heading whose text is the entry
+    VERBATIM, separator lines included — a fallback that partially cleaned its
+    input would be a second transformation nobody reviewed.
+
+    Coverage invariant (§2.4, and the test): every line of the source aggregate
+    is either a discarded separator in a decomposed entry, or appears in exactly
+    one heading, exactly once. No line is dropped, none is duplicated, and no
+    path both extracts headings and re-renders the original text.
+
+    Total over its input: a non-string entry is skipped rather than raised on, so
+    no provider shape can fail detail assembly. The all-or-nothing rule for a
+    malformed payload is enforced at the call site, which validates the whole
+    entry list before calling.
+    """
+    out: list[str] = []
+    for entry in entries or ():
+        if not isinstance(entry, str) or not entry.strip():
+            continue
+        lines = [line.strip() for line in entry.split("\n") if line.strip()]
+        kept = [line for line in lines if line != _SEPARATOR_LINE]
+        matches = [_HEADING_LINE_RE.fullmatch(line) for line in kept]
+        if kept and all(match is not None for match in matches):
+            out.extend(match.group(1) for match in matches)
+        else:
+            out.append(entry)
+    return out