@yahyahammad/venastine 0.1.0 → 0.3.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/.env.example +19 -14
  2. package/CONFIG_ARCHITECTURE.md +2002 -0
  3. package/NOTICE +1 -1
  4. package/README.md +249 -93
  5. package/agents/builtin/build.md +73 -0
  6. package/agents/builtin/general.md +80 -0
  7. package/agents/builtin/plan.md +23 -5
  8. package/agents/builtin/test.md +65 -0
  9. package/agents/builtin/writer.md +67 -0
  10. package/agents/manager.py +261 -195
  11. package/agents/subagent_tool.py +280 -23
  12. package/agents/tui_commands.py +31 -21
  13. package/bin/venastine.mjs +528 -382
  14. package/bin/venastine.py +71 -0
  15. package/cli_commands.py +835 -0
  16. package/config.py +95 -1198
  17. package/config.yaml +1090 -0
  18. package/config_edit.py +1301 -0
  19. package/config_schema.py +966 -0
  20. package/config_update.py +749 -0
  21. package/core/agent_activity.py +256 -0
  22. package/core/ansi.py +452 -0
  23. package/core/approval.py +27 -5
  24. package/core/call_view.py +364 -0
  25. package/core/cancel.py +121 -0
  26. package/core/chatgpt_auth.py +747 -0
  27. package/core/claims_view.py +107 -0
  28. package/core/client.py +1249 -82
  29. package/core/commands.py +454 -0
  30. package/core/compaction.py +1735 -959
  31. package/core/config_loader.py +516 -97
  32. package/core/connect.py +686 -0
  33. package/core/events.py +68 -6
  34. package/core/file_mentions.py +280 -0
  35. package/core/interaction.py +87 -4
  36. package/core/line_pattern.py +128 -0
  37. package/core/loop.py +2566 -1353
  38. package/core/memory.py +482 -370
  39. package/core/pipeline_models.py +23 -9
  40. package/core/provider_errors.py +234 -0
  41. package/core/reasoning/confidence_scoring.py +70 -54
  42. package/core/reasoning/json_retry.py +19 -1
  43. package/core/reasoning/orchestrator.py +232 -53
  44. package/core/reasoning/output_writer.py +90 -31
  45. package/core/reasoning/payload_validation.py +133 -7
  46. package/core/reasoning/pipeline_storage.py +427 -361
  47. package/core/reasoning/review.py +659 -615
  48. package/core/reasoning/scholar.py +38 -14
  49. package/core/reasoning/source_corpus.py +110 -4
  50. package/core/replay.py +315 -167
  51. package/core/session.py +36 -12
  52. package/core/session_view.py +259 -0
  53. package/core/session_wake.py +259 -0
  54. package/core/shell_sessions.py +1553 -0
  55. package/core/workspace_trust.py +1 -1
  56. package/credentials.py +308 -180
  57. package/database.py +82 -1
  58. package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
  59. package/docs/SECURITY.md +108 -0
  60. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  61. package/env_secrets.py +118 -77
  62. package/json_store.py +236 -16
  63. package/logging_setup.py +121 -11
  64. package/main.py +1672 -198
  65. package/mcp_client/client.py +257 -23
  66. package/mcp_client/config.py +484 -379
  67. package/mcp_client/registration.py +163 -2
  68. package/memories/manager.py +15 -0
  69. package/memories/tui_commands.py +21 -14
  70. package/package.json +303 -73
  71. package/project_init/config_files.py +4 -0
  72. package/project_init/generator.py +201 -37
  73. package/project_init/manifest.py +418 -406
  74. package/project_init/tui_commands.py +7 -4
  75. package/prompts/revalidate.md +1 -1
  76. package/prompts/revise.md +17 -15
  77. package/prompts/system_prompts.py +64 -10
  78. package/providers.json.example +94 -92
  79. package/pyproject.toml +149 -100
  80. package/requirements.txt +252 -29
  81. package/safety/policy_enforcement.py +1082 -682
  82. package/security/askpass.py +419 -0
  83. package/security/capability.py +37 -7
  84. package/security/confine.py +484 -0
  85. package/security/hosted.py +182 -0
  86. package/security/permissions.py +54 -7
  87. package/security/posture.py +124 -12
  88. package/security/protected_paths.py +198 -6
  89. package/security/sandbox.py +3262 -139
  90. package/security/secrets.py +855 -0
  91. package/skills/tui_commands.py +3 -4
  92. package/storage.py +1338 -1005
  93. package/tools/base.py +465 -305
  94. package/tools/builtin/_math_common.py +306 -16
  95. package/tools/builtin/_net_common.py +267 -61
  96. package/tools/builtin/arxiv.py +681 -223
  97. package/tools/builtin/ask_user.py +231 -213
  98. package/tools/builtin/discrete_math.py +35 -4
  99. package/tools/builtin/fetch_url.py +239 -21
  100. package/tools/builtin/file_ops.py +341 -33
  101. package/tools/builtin/geometry.py +184 -148
  102. package/tools/builtin/get_time.py +2 -2
  103. package/tools/builtin/linear_algebra.py +66 -23
  104. package/tools/builtin/load_skill.py +10 -0
  105. package/tools/builtin/logic.py +171 -12
  106. package/tools/builtin/pin.py +46 -12
  107. package/tools/builtin/probability_stats.py +224 -228
  108. package/tools/builtin/project_docs.py +80 -21
  109. package/tools/builtin/remember.py +21 -4
  110. package/tools/builtin/shell.py +586 -67
  111. package/tools/builtin/shell_sessions.py +613 -0
  112. package/tools/builtin/symbolic_math.py +80 -10
  113. package/tools/builtin/todo.py +5 -3
  114. package/tools/builtin/web_search.py +329 -151
  115. package/tools/context.py +23 -0
  116. package/tools/isolation.py +387 -314
  117. package/tools/registry.py +374 -29
  118. package/tui/app.py +5281 -397
  119. package/tui/app.tcss +290 -9
  120. package/tui/commands.py +26 -61
  121. package/tui/diffs.py +286 -265
  122. package/tui/history.py +128 -0
  123. package/tui/hosted_driver.py +102 -0
  124. package/tui/markdown.py +1329 -0
  125. package/tui/meters.py +364 -0
  126. package/tui/screens.py +346 -92
  127. package/tui/themes.py +599 -366
  128. package/tui/widgets.py +2673 -116
  129. package/SECURITY.md +0 -88
  130. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -0,0 +1,1329 @@
1
+ """
2
+ tui/markdown.py
3
+
4
+ What a markdown answer is made of, as data (batch 53).
5
+
6
+ The transcript used to render exactly one construct -- a ``` fence, through
7
+ `rich.syntax.Syntax` -- and hand everything else to `Text` unchanged. So a
8
+ table arrived as its own pipes, and `_split_fences`' docstring said why:
9
+ "deliberately small: this is a renderer, not a markdown parser." This module
10
+ is the parser that sentence declined to write, kept to the constructs a
11
+ reader actually loses by not having: a TABLE, a HEADING, the inline marks
12
+ (`**strong**`, `` `code` ``, and batch 58's `*emphasis*` and `~~strike~~`), a
13
+ LIST ITEM, and a bare URL.
14
+
15
+ CURATED rather than complete, and the omissions are decisions with reasons
16
+ recorded in AGENTS.md: no block quotes (a bar down the left collides with the
17
+ thinking span's own bar), no horizontal rules, no heading LEVELS, no setext
18
+ headings, nothing inside a reasoning span.
19
+
20
+ PURE. No I/O, no `core/` import, no harness state -- `tui/diffs.py`'s rule,
21
+ because `tui/widgets.py` imports this the same way. Parsing lives here;
22
+ building the Rich renderables stays in the widget, so a replay under a new
23
+ theme re-derives the painting rather than replaying something decorated once.
24
+
25
+ INSIDE the `tui` package on purpose. A root-level `markdown.py` would sit on
26
+ `sys.path` beside the interpreter's own imports -- the `logging.py` incident,
27
+ and the reason `mcp_client/` is not called `mcp/`. Nothing here shadows
28
+ `rich.markdown`, which is imported absolutely and by nobody in this project.
29
+
30
+ WHAT THIS OWNS, and the last one is the one that is easy to miss.
31
+
32
+ - `split_blocks` -- plain text, fenced code, tables.
33
+ - `verbatim` / `heading` / `list_item` -- what one LINE is.
34
+ - `inline_spans` -- one line as `(text, role)` pairs.
35
+ - `wrap_display` -- the rows a list item's body will occupy.
36
+ - `commit_span` -- how far a streamed answer may be drawn RIGHT NOW, and
37
+ whether the cap gave up. `safe_commit_limit` is its first value.
38
+
39
+ That third one is the whole reason the first two are safe. `Transcript`
40
+ commits a streamed answer at boundaries it computes itself, and RichLog
41
+ cannot rewrite a row it has drawn -- so a construct must never be committed
42
+ in halves. §38 shipped one instance of that rule (an open fence is held);
43
+ this generalises it to a CAP: the offset past which committing would split
44
+ something. FIVE constructs answer to it, listed at the function, and batch 58
45
+ put a ceiling on the three of them that are waiting for a newline no model is
46
+ obliged to send.
47
+
48
+ MEASUREMENT IS THE OTHER HALF, and the two are useless apart. `**bold**`
49
+ renders four cells shorter than its source, so a cut taken in SOURCE columns
50
+ lands where Rich would not have wrapped -- and the streamed rows then differ
51
+ from the ones `rerender()` draws from the same text, which is the divergence
52
+ `TestAStreamedAnswerRendersLikeAWrittenOne` exists to catch. `display_len`
53
+ and `width_split` measure what will actually be on screen; the cap is what
54
+ makes that measurable, because no cut is ever offered while a mark is still
55
+ open and its rendered width therefore still unknown.
56
+
57
+ Cells rather than characters (`rich.cells.cell_len`): Rich wraps by cells,
58
+ so this is the same question Rich will ask, and a double-width glyph is
59
+ counted the way the terminal will draw it.
60
+ """
61
+
62
+ import re
63
+
64
+ from rich.cells import cell_len
65
+
66
+ #: The fence delimiter, spelled once. `Transcript._fence_is_open` counts
67
+ #: these and this module splits on them, and two spellings of one token is
68
+ #: the shape this project keeps finding drift in.
69
+ FENCE = "```"
70
+
71
+ #: Roles for the marks, resolved against the active theme by
72
+ #: `tui/themes.role_styles`. Strings rather than an enum for the reason the
73
+ #: transcript's own roles are strings: they are palette keys, and the
74
+ #: palette is a dict.
75
+ HEADING = "md_heading"
76
+ STRONG = "md_strong"
77
+ CODE = "md_code"
78
+ #: Batch 58. `LINK` is the one role whose span TEXT carries information: a
79
+ #: link span is always the URL itself, so the renderer needs no second
80
+ #: channel to know what a click should open. That is the security rule
81
+ #: spelled as a data shape -- there is no label to hide a target behind.
82
+ EM = "md_em"
83
+ STRIKE = "md_strike"
84
+ LINK = "md_link"
85
+ BULLET = "md_bullet"
86
+
87
+ #: The ceiling on a hold (batch 58). `safe_commit_limit` holds a construct
88
+ #: rather than commit it in halves, and three of those holds are waiting on
89
+ #: a newline that a model is under no obligation to send. Past this many
90
+ #: characters the construct stops being one -- on BOTH paths, which is what
91
+ #: keeps the streamed rows equal to the replayed ones -- and the stream is
92
+ #: released. A fence and a table are exempt and say why at the function.
93
+ #:
94
+ #: Characters rather than rows, deliberately: a rule measured in rows would
95
+ #: depend on the terminal, so a resize or a `/theme` could re-classify a
96
+ #: line and `rerender()` would draw something the live path never drew.
97
+ HOLD_LIMIT = 1000
98
+
99
+ #: Indented at or past this, a line is drawn exactly as it was written.
100
+ #: Markdown's own code-block indent, used here only as a NO-PARSE zone --
101
+ #: this module does not render indented code as a block, it declines to
102
+ #: read anything into one. Fences are exempt; see TECHNICAL_DEBT.md.
103
+ CODE_INDENT = 4
104
+
105
+ #: ATX headings only, and the space is REQUIRED -- `#hashtag` is not a
106
+ #: heading, and `###` alone is not one either. Closing hashes are left
107
+ #: alone: GFM strips them, and a rule that eats trailing `#` characters
108
+ #: would silently edit a line that merely ends in one.
109
+ _HEADING = re.compile(r"^(#{1,6})\s+")
110
+
111
+ #: One cell of a GFM delimiter row: dashes, optionally anchored by a colon
112
+ #: at either end to set the column's alignment.
113
+ _DELIMITER_CELL = re.compile(r"^:?-+:?$")
114
+
115
+ #: Everything a delimiter row may be built from. Used to decide, from a
116
+ #: PARTIAL next line, whether a table is still possible -- one character
117
+ #: outside this set settles it, which is what keeps a paragraph following
118
+ #: a pipe in prose from being held until its own newline arrives.
119
+ _DELIMITER_CHARS = set("|-: \t")
120
+
121
+ #: A list item's opener: bullet or ordered, then ONE to FOUR spaces. The
122
+ #: upper bound is GFM's -- past four the content is an indented block
123
+ #: rather than the item's text -- and it also stops a stray `- ` in a
124
+ #: column of ASCII art from claiming a hanging indent it cannot fill.
125
+ #: The leading run is unbounded here because `verbatim()` is what rules
126
+ #: out an indented line, so CODE_INDENT stays the single spelling of that.
127
+ _LIST = re.compile(r"^( *)([-*+]|\d{1,9}[.)])( {1,4})(?! )(.*)$")
128
+
129
+ #: A bare URL, and the only thing in this grammar that becomes clickable.
130
+ #: Ends at whitespace or at a character that cannot appear in one without
131
+ #: being escaped -- the brackets of a markdown link are NOT excluded, so
132
+ #: `[docs](https://x/a)` yields the URL and leaves the label as prose.
133
+ _URL = re.compile(r"https?://[^\s<>\"'`\\]+")
134
+
135
+ #: Trailing characters trimmed off a detected URL: sentence punctuation
136
+ #: and the marks that sit beside one in prose. A closing bracket is
137
+ #: handled separately, by BALANCE, so `https://x/a_(b)` keeps its own.
138
+ _URL_TRIM = ".,;:!?*_~'\""
139
+
140
+ #: Bracket pairs whose closer is trimmed only when it was never opened.
141
+ _URL_PAIRS = {")": "(", "]": "[", "}": "{"}
142
+
143
+ #: What `safety.policy_enforcement` leaves where a secret was, and the
144
+ #: one thing this module knows about that one. Its own copy rather than
145
+ #: an import, because this file is pure -- it imports `re` and nothing
146
+ #: else -- and reaching into policy for a nine-character string would
147
+ #: cost that for no gain. `tests/test_markdown_render.py` holds the two
148
+ #: against each other, so the duplication is checked rather than
149
+ #: trusted.
150
+ REDACTED_MARK = "[REDACTED]"
151
+
152
+
153
+ # ---------------------------------------------------------------------------
154
+ # ---- Lines ----------------------------------------------------------------
155
+ # ---------------------------------------------------------------------------
156
+
157
+ def verbatim(line: str) -> bool:
158
+ """Is this line drawn exactly as it was written (batch 58)?
159
+
160
+ Four columns of indent, or any tab in the indent, and NOTHING is read
161
+ into the line: not a list marker, not a table row, not an inline mark.
162
+ A reported defect rather than a tidy-up -- `x = a ** b ** c` indented
163
+ inside an answer rendered with ` b ` in bold, because batch 53 parsed
164
+ marks on every line regardless of where it sat.
165
+
166
+ Stateless on purpose. GFM decides this with the surrounding list
167
+ context; tracking that would put state across the commit boundary,
168
+ which is where this widget has broken before. The price is named
169
+ rather than hidden: an item nested with FOUR spaces renders flat, and
170
+ one nested with two renders as a nested item.
171
+
172
+ A heading was never exposed -- `_HEADING` is anchored at column 0 --
173
+ and a FENCE is deliberately still exempt. See TECHNICAL_DEBT.md.
174
+ """
175
+ indent = line[:len(line) - len(line.lstrip())]
176
+ return "\t" in indent or len(indent) >= CODE_INDENT
177
+
178
+
179
+ def heading(line: str):
180
+ """The end of this line's ATX marker, or None if it has none.
181
+
182
+ One reading for the renderer and the cap, so a heading held by one
183
+ cannot be drawn as prose by the other. The HOLD_LIMIT guard is half
184
+ of that agreement: a `#` line longer than the limit is not a heading
185
+ on either path, so the released chunk and the replay draw the same
186
+ hashes.
187
+ """
188
+ if len(line) > HOLD_LIMIT or verbatim(line):
189
+ return None
190
+ match = _HEADING.match(line)
191
+ return match.end() if match else None
192
+
193
+
194
+ def list_item(line: str):
195
+ """`(prefix, body, indent)` for a list item, or None.
196
+
197
+ `prefix` is everything before the text -- indent, marker and the gap
198
+ -- and `indent` is the column the text starts at, which is where the
199
+ item's own wrapped rows are padded to. One number for both, because a
200
+ hanging indent that did not line up with the marker's own text would
201
+ be furniture rather than structure.
202
+
203
+ Refused past HOLD_LIMIT for `heading`'s reason, and it is the half
204
+ that makes the stall bounded: a marker with no newline behind it is
205
+ held only until the limit, and past it the line is prose on BOTH
206
+ paths rather than a list item on one of them.
207
+ """
208
+ if verbatim(line) or len(line) > HOLD_LIMIT:
209
+ return None
210
+ match = _LIST.match(line)
211
+ if match is None:
212
+ return None
213
+ prefix = line[:match.end(3)]
214
+ return prefix, match.group(4), len(prefix)
215
+
216
+
217
+ # ---------------------------------------------------------------------------
218
+ # ---- Blocks ---------------------------------------------------------------
219
+ # ---------------------------------------------------------------------------
220
+
221
+ class CodeBlock(tuple):
222
+ """A fenced block: `(language, code)`, language None when unlabelled.
223
+
224
+ A tuple subclass rather than a NamedTuple or a dataclass, and that is
225
+ load-bearing rather than lazy. `Transcript._render_blocks` asks
226
+ `isinstance(block, tuple)` to mean "a renderable that occupies its own
227
+ rows", which is what drives the newline trimming around it -- so both
228
+ block kinds answering that question keeps ONE rule for the two of them
229
+ instead of a second branch that has to be kept in step. It also unpacks
230
+ exactly as §26's plain `(language, code)` tuple did.
231
+ """
232
+
233
+ __slots__ = ()
234
+
235
+ def __new__(cls, language, code):
236
+ return super().__new__(cls, (language, code))
237
+
238
+ @property
239
+ def language(self):
240
+ return self[0]
241
+
242
+ @property
243
+ def code(self):
244
+ return self[1]
245
+
246
+
247
+ class TableBlock(tuple):
248
+ """A GFM table: `(headers, aligns, rows)`.
249
+
250
+ `aligns` is one of "left" / "center" / "right" per column, in Rich's own
251
+ vocabulary so the renderer hands it straight to `Table.add_column` --
252
+ translating at the boundary rather than storing a markdown spelling the
253
+ widget would have to re-map.
254
+
255
+ Rows are RAGGED-SAFE: short ones are padded and long ones truncated to
256
+ the header's width, which is what GFM specifies and, more to the point,
257
+ what stops one malformed row from raising inside a render.
258
+ """
259
+
260
+ __slots__ = ()
261
+
262
+ def __new__(cls, headers, aligns, rows):
263
+ return super().__new__(cls, (headers, aligns, rows))
264
+
265
+ @property
266
+ def headers(self):
267
+ return self[0]
268
+
269
+ @property
270
+ def aligns(self):
271
+ return self[1]
272
+
273
+ @property
274
+ def rows(self):
275
+ return self[2]
276
+
277
+
278
+ def _is_fence_line(line: str) -> bool:
279
+ """Does this line open or close a fenced block?
280
+
281
+ A fence starts the LINE (after whitespace). A ``` mentioned mid-line
282
+ (`Use ```python to start`) is prose mentioning a fence, not a block.
283
+ Any indent counts -- fences are deliberately exempt from `verbatim()`,
284
+ so an indented fence still opens (confirmed, not GFM-strict).
285
+ """
286
+ return line.lstrip(" \t").startswith(FENCE)
287
+
288
+
289
+ def _fence_line_count(text: str) -> int:
290
+ """How many fence-lines `text` holds. `commit_span`'s parity check,
291
+ kept beside the splitter so the two cannot drift into substring
292
+ counting on one path and line counting on the other."""
293
+ return sum(1 for raw in text.splitlines()
294
+ if _is_fence_line(raw))
295
+
296
+
297
+ def split_blocks(text: str):
298
+ """Split markdown into plain strings, `CodeBlock`s and `TableBlock`s.
299
+
300
+ §26's `_split_fences` with a second pass inside its plain parts, and the
301
+ fence half keeps its contract -- an unterminated fence still runs to
302
+ the end of the text (the common case mid-stream), and the plain parts
303
+ still carry their own newlines so the caller's trimming around a
304
+ renderable block keeps working on the reconstructed text.
305
+
306
+ Tables are looked for only OUTSIDE fences, which is why this is nested
307
+ rather than a scan over lines: a pipe table inside a ``` block is source
308
+ code, and highlighting it as one is the whole point of the fence.
309
+
310
+ Fences are LINE constructs (`_is_fence_line`): a ``` mid-line is prose.
311
+ """
312
+ out, plain, language, code = [], [], None, None
313
+ in_code = False
314
+ for raw in text.splitlines(keepends=True):
315
+ line = _strip_eol(raw)
316
+ if _is_fence_line(line):
317
+ if not in_code:
318
+ if plain:
319
+ out.extend(_split_tables("".join(plain)))
320
+ plain = []
321
+ language = line.lstrip(" \t")[len(FENCE):].strip() or None
322
+ code = []
323
+ in_code = True
324
+ else:
325
+ out.append(CodeBlock(language, "".join(code)))
326
+ language, code, in_code = None, None, False
327
+ if raw.endswith("\n"):
328
+ # The old `text.split(FENCE)` left the closing line's
329
+ # newline in the FOLLOWING plain run, and the
330
+ # renderer's blank-line trimming depends on it.
331
+ plain.append("\n")
332
+ continue
333
+ (code if in_code else plain).append(raw)
334
+ if in_code:
335
+ out.append(CodeBlock(language, "".join(code)))
336
+ elif plain:
337
+ out.extend(_split_tables("".join(plain)))
338
+ return out
339
+
340
+
341
+ def _split_tables(text: str):
342
+ """One fence-free stretch as plain strings and `TableBlock`s.
343
+
344
+ The plain runs are re-joined with their original line endings, so
345
+ `"".join(the plain parts) + the tables' own source` is the input back --
346
+ which is what lets the caller's blank-line trimming treat a table
347
+ exactly as it treats a fence.
348
+
349
+ THE LAST ROW'S NEWLINE IS LEFT BEHIND, and that is what makes "exactly"
350
+ true rather than approximately. `text.split(FENCE)` puts the newline
351
+ after a closing ``` into the FOLLOWING plain run, so the trimming finds
352
+ two there and keeps one -- a fence draws a blank line after it. A table
353
+ that swallowed its own terminator left only one newline behind, the
354
+ trimming took it, and the paragraph after a table butted straight up
355
+ against the bottom border while the same paragraph after a fence did
356
+ not. Handing the newline back puts both constructs on one rule.
357
+ """
358
+ lines = text.splitlines(keepends=True)
359
+ out, plain, index = [], [], 0
360
+ while index < len(lines):
361
+ table, consumed = _read_table(lines, index)
362
+ if table is None:
363
+ plain.append(lines[index])
364
+ index += 1
365
+ continue
366
+ if plain:
367
+ out.append("".join(plain))
368
+ plain = []
369
+ out.append(table)
370
+ if lines[index + consumed - 1].endswith("\n"):
371
+ plain.append("\n")
372
+ index += consumed
373
+ if plain:
374
+ out.append("".join(plain))
375
+ return out
376
+
377
+
378
+ def _read_table(lines, start):
379
+ """`(TableBlock, lines consumed)` if a table starts at `start`, else
380
+ `(None, 0)`.
381
+
382
+ GFM-STRICT, which is the decision rather than an implementation detail:
383
+ a header row is only a header when a delimiter row follows it AND the two
384
+ have the same number of cells. Without that, a shell pipeline, a `|` in
385
+ prose or a column of BNF alternatives all become tables -- and, worse
386
+ than rendering oddly, each one would stall the stream while the cap below
387
+ waited for a table that was never coming.
388
+ """
389
+ header = _strip_eol(lines[start])
390
+ if not _is_row(header):
391
+ return None, 0
392
+ if start + 1 >= len(lines):
393
+ return None, 0
394
+ delimiter = _strip_eol(lines[start + 1])
395
+ headers = _cells(header)
396
+ aligns = _delimiter_aligns(delimiter, len(headers))
397
+ if aligns is None:
398
+ return None, 0
399
+
400
+ rows, index = [], start + 2
401
+ while index < len(lines):
402
+ body = _strip_eol(lines[index])
403
+ if not _is_row(body):
404
+ break
405
+ rows.append(_fit(_cells(body), len(headers)))
406
+ index += 1
407
+ return TableBlock(headers, aligns, rows), index - start
408
+
409
+
410
+ def _strip_eol(line: str) -> str:
411
+ return line[:-1] if line.endswith("\n") else line
412
+
413
+
414
+ def _is_row(text: str) -> bool:
415
+ """A line that could be part of a table: non-blank, and carrying a pipe.
416
+
417
+ A blank line ends a table, which is GFM's rule and also the only thing
418
+ that keeps a table from swallowing the paragraph after it.
419
+
420
+ An INDENTED line is not a row (batch 58), which is what keeps a shell
421
+ pipeline inside a four-space code sample from being drawn as a grid.
422
+ Here rather than at the two callers, so `_read_table` and the cap
423
+ cannot disagree about where a table stops.
424
+ """
425
+ return bool(text.strip()) and "|" in text and not verbatim(text)
426
+
427
+
428
+ def _trailing_backslashes(text: str) -> int:
429
+ """Count of trailing backslashes in `text`. The parity decides whether
430
+ a final pipe is escaped (odd) or a delimiter (even)."""
431
+ count = 0
432
+ for char in reversed(text):
433
+ if char != "\\":
434
+ break
435
+ count += 1
436
+ return count
437
+
438
+
439
+ def _cells(text: str):
440
+ """One row's cells. Outer pipes are optional (GFM allows a table with
441
+ none at all), and `\\|` inside a cell is a literal pipe rather than a
442
+ boundary -- unescaped here so the cell carries what the model meant.
443
+
444
+ Backslash parity, left to right: a double backslash is a literal
445
+ backslash (checked first), so double-backslash-pipe is that backslash
446
+ followed by a DELIMITER, while triple-backslash-pipe is a backslash
447
+ followed by an escaped pipe and stays one cell.
448
+ """
449
+ body = text.strip()
450
+ if body.startswith("|"):
451
+ body = body[1:]
452
+ if body.endswith("|") and _trailing_backslashes(body[:-1]) % 2 == 0:
453
+ body = body[:-1]
454
+ cells, current, index = [], [], 0
455
+ while index < len(body):
456
+ char = body[index]
457
+ nxt = body[index + 1] if index + 1 < len(body) else ""
458
+ if char == "\\" and nxt == "\\":
459
+ current.append("\\")
460
+ index += 2
461
+ continue
462
+ if char == "\\" and nxt == "|":
463
+ current.append("|")
464
+ index += 2
465
+ continue
466
+ if char == "|":
467
+ cells.append("".join(current).strip())
468
+ current = []
469
+ index += 1
470
+ continue
471
+ current.append(char)
472
+ index += 1
473
+ cells.append("".join(current).strip())
474
+ return cells
475
+
476
+
477
+ def _delimiter_aligns(text: str, columns: int):
478
+ """The alignments a delimiter row declares, or None if it is not one.
479
+
480
+ The column-count match is part of "is this a delimiter row" rather than
481
+ a separate validation, because that is the question the caller is
482
+ asking: a `|---|` under a three-column header is not a table with a
483
+ mistake in it, it is not a table.
484
+ """
485
+ if not text.strip():
486
+ return None
487
+ cells = _cells(text)
488
+ if len(cells) != columns:
489
+ return None
490
+ aligns = []
491
+ for cell in cells:
492
+ if not _DELIMITER_CELL.match(cell):
493
+ return None
494
+ left, right = cell.startswith(":"), cell.endswith(":")
495
+ if left and right:
496
+ aligns.append("center")
497
+ elif right:
498
+ aligns.append("right")
499
+ else:
500
+ aligns.append("left")
501
+ return aligns
502
+
503
+
504
+ def _fit(cells, columns):
505
+ if len(cells) < columns:
506
+ return cells + [""] * (columns - len(cells))
507
+ return cells[:columns]
508
+
509
+
510
+ def _delimiter_possible(prefix: str) -> bool:
511
+ """Could this PARTIAL line still turn into a delimiter row?
512
+
513
+ The cheap half of the decision the cap needs. A delimiter row is built
514
+ from four characters; one outside that set proves the line above it was
515
+ prose, and the stream can be released without waiting for the newline.
516
+ Deciding this from a prefix is what keeps a `|` inside a sentence from
517
+ holding the whole paragraph that follows it.
518
+ """
519
+ return all(char in _DELIMITER_CHARS for char in prefix)
520
+
521
+
522
+ # ---------------------------------------------------------------------------
523
+ # ---- Inline marks ---------------------------------------------------------
524
+ # ---------------------------------------------------------------------------
525
+
526
+ def inline_spans(line: str, *, block: bool = True):
527
+ """One line as `(text, role)` pairs; role is "" for unmarked text.
528
+
529
+ Deliberately smaller than GFM, and each omission is a decision:
530
+
531
+ - `__strong__` is NOT recognised, and batch 58 did not reverse it.
532
+ Measured against a real CommonMark parser, `call __init__ on it`
533
+ renders a bold `init` under the full rules, anywhere in the line,
534
+ and no flanking rule saves it. `_x_` goes with it: an underscore
535
+ that cannot be trusted at two is not trusted at one either.
536
+ - `*emphasis*` IS recognised as of batch 58, but only where the
537
+ asterisk is not wedged between two word characters. That is what
538
+ keeps `2*3*4` arithmetic and `x*y*z` a filename -- both of which
539
+ full GFM italicises -- while `the *point*` renders.
540
+ - a code span is delimited by single backticks and cannot span a line,
541
+ which is what every model this harness talks to actually emits.
542
+ - a URL is its own span and carries no label. `[docs](https://x)`
543
+ draws every character it was written with; only the URL is marked.
544
+
545
+ `block=False` says this text is a FRAGMENT of a line rather than a
546
+ line -- the tail of a streamed paragraph, or a table cell. Nothing
547
+ that depends on where a line STARTS may fire there: not a heading,
548
+ not the indent rule. It is the fix for a shipped defect where a wrap
549
+ boundary falling just before a `# ` token made the committed fragment
550
+ render as a heading and swallow the hash, which a `/theme` replay then
551
+ put back.
552
+
553
+ A heading contributes its role to the line's UNMARKED spans and leaves
554
+ marked ones alone, so `# The **why**` keeps the strong mark visible
555
+ inside the heading rather than compositing two styles into a third.
556
+ """
557
+ base = ""
558
+ body = line
559
+ end = heading(line) if block else None
560
+ if end is not None:
561
+ body, base = line[end:], HEADING
562
+ return [(text, role or base)
563
+ for text, role, _s, _e in _spans(body, block=block)]
564
+
565
+
566
+ def link_spans(text: str, *, targets=()):
567
+ """`(text, role, target)` triples where the ONLY mark is a URL.
568
+
569
+ The scanner a HARNESS-DRAWN line gets (batch 65), where
570
+ `inline_spans` is the one model prose gets. A tool line is a
571
+ digest -- a shell command, a JSON argument -- not prose, and the
572
+ full grammar does not merely over-decorate it, it EATS
573
+ CHARACTERS. Measured on real calls: `git log --format=%h # `date``
574
+ loses its backticks to a code span, and an MCP filter
575
+ `name~=*test* ~~old~~` renders as `name~=test old`. A digest that
576
+ no longer shows what was run is worse than an inert URL, so
577
+ everything that is not a URL passes through literally.
578
+
579
+ `target` is what a click OPENS and is normally the span's own text,
580
+ which is batch 58's rule unchanged: the URL is its own label, so
581
+ there is nowhere for a destination to differ from what is read.
582
+
583
+ `targets` is the exception, and it exists because a tool line is
584
+ TRUNCATED before it is drawn -- `param_digest` caps a value at 60
585
+ characters, which is shorter than most real documentation URLs.
586
+ An elided run resolves against these candidates, and three things
587
+ have to hold at once:
588
+
589
+ - the run is elided at all, i.e. it ends in characters that are
590
+ not ASCII. Asked that way rather than by naming the ellipsis,
591
+ so this module needs to know nothing about the producer that
592
+ put one there;
593
+ - exactly ONE candidate has the visible part as its prefix. Two
594
+ would make the click a guess;
595
+ - THE WHOLE ORIGIN IS VISIBLE. The invariant that replaces
596
+ "the visible text is the target" is `the visible text tells
597
+ you the origin, and the origin is where it goes`, so a run cut
598
+ off inside its host resolves to nothing and stays literal.
599
+ Truncation may elide a path and a query; it may never elide
600
+ the answer to "where does this take me".
601
+
602
+ A candidate is itself run through `_url`, so `clickable`'s rules --
603
+ ASCII, http(s), and no userinfo -- gate a target exactly as they
604
+ gate a span, and a caller cannot hand in something the reader
605
+ would not have been allowed to click.
606
+ """
607
+ candidates = [url for value in targets for url in _urls_in(value)]
608
+ out = []
609
+ cursor = at = 0
610
+ while True:
611
+ at = text.find("http", at)
612
+ if at == -1:
613
+ break
614
+ run = _URL.match(text, at)
615
+ if run is None or (at and text[at - 1].isalnum()):
616
+ at += 1
617
+ continue
618
+ url = _trim_url(run.group(0))
619
+ target = url if clickable(url) else _elided(url, candidates)
620
+ if not target:
621
+ at += 1
622
+ continue
623
+ if at > cursor:
624
+ out.append((text[cursor:at], "", ""))
625
+ out.append((url, LINK, target))
626
+ cursor = at = at + len(url)
627
+ if cursor < len(text):
628
+ out.append((text[cursor:], "", ""))
629
+ return out
630
+
631
+
632
+ def _urls_in(value: str):
633
+ """Every armable URL inside one candidate value.
634
+
635
+ Callers hand in whole PARAMETER VALUES rather than URLs, so the URL
636
+ grammar has one definition and it is this module's. A search query
637
+ that happens to contain a URL therefore offers it, and a value that
638
+ contains none offers nothing.
639
+ """
640
+ if not isinstance(value, str):
641
+ return []
642
+ found = []
643
+ at = 0
644
+ while True:
645
+ at = value.find("http", at)
646
+ if at == -1:
647
+ return found
648
+ span, _opened = _url(value, at)
649
+ if span is None:
650
+ at += 1
651
+ continue
652
+ found.append(span[0])
653
+ at = span[3]
654
+
655
+
656
+ def _elided(run: str, candidates) -> str:
657
+ """The full URL a truncated `run` stands for, or "".
658
+
659
+ See `link_spans`. The three conditions are in the order that makes
660
+ the cheapest one fail first, and the last of them is the one that
661
+ carries the security property.
662
+ """
663
+ end = len(run)
664
+ while end and not run[end - 1].isascii():
665
+ end -= 1
666
+ if end == len(run) or not end:
667
+ return ""
668
+ prefix = run[:end]
669
+ matched = [url for url in candidates if url.startswith(prefix)]
670
+ if len(matched) != 1:
671
+ return ""
672
+ target = matched[0]
673
+ origin = len(target) - len(target.split("://", 1)[-1]) \
674
+ + len(_authority(target))
675
+ return target if len(prefix) >= origin else ""
676
+
677
+
678
+ def _spans(text: str, *, block: bool = True):
679
+ """`(text, role, source start, source end)` for one line."""
680
+ return _scan(text, block=block)[0]
681
+
682
+
683
+ def _wordish(char: str) -> bool:
684
+ return bool(char) and (char.isalnum() or char == "_")
685
+
686
+
687
+ def _em_edge(text: str, index: int, *, opening: bool) -> bool:
688
+ """May the `*` at `index` be that end of an emphasis span?
689
+
690
+ Two rules, and each answers one of batch 53's two objections.
691
+
692
+ WEDGED between two word characters, it is arithmetic or a filename
693
+ and never italics -- measured against a real CommonMark parser, full
694
+ GFM renders `2*3*4` and `x*y*z` with emphasis and this declines to.
695
+
696
+ And an opener must be followed by a non-space, a closer preceded by
697
+ one, which is GFM's own flanking rule and the half that matters for
698
+ STREAMING: without it `a * b * c` would open a mark that never
699
+ closes, and the cap would hold the rest of the paragraph waiting for
700
+ a closer that is not coming.
701
+ """
702
+ before = text[index - 1] if index else ""
703
+ after = text[index + 1] if index + 1 < len(text) else ""
704
+ if _wordish(before) and _wordish(after):
705
+ return False
706
+ edge = after if opening else before
707
+ return bool(edge) and not edge.isspace()
708
+
709
+
710
+ def _paired(text: str, index: int, token: str, role: str):
711
+ """`(span, opened)` for a `**`/`~~`/`` ` `` mark starting at `index`.
712
+
713
+ `span` when it closes on this line; `opened` ONLY when nothing closes
714
+ it at all, because that is the one case the cap can usefully hold --
715
+ an empty mark (`****`) is literal and finished, and so is a closer
716
+ further away than HOLD_LIMIT, which is the rule that keeps the
717
+ released stream and the replay drawing the same asterisks.
718
+ """
719
+ size = len(token)
720
+ close = text.find(token, index + size)
721
+ if close == -1:
722
+ return None, True
723
+ if close > index + size and close - index <= HOLD_LIMIT:
724
+ return (text[index + size:close], role, index, close + size), False
725
+ return None, False
726
+
727
+
728
+ def _em(text: str, index: int):
729
+ """`(span, opened)` for a `*emphasis*` span starting at `index`.
730
+
731
+ A loop rather than a `find`, because a candidate closer can be
732
+ refused: `a *b*c` has an asterisk that cannot close, so the mark is
733
+ still OPEN and the cap must keep holding -- the closer may be the one
734
+ that has not arrived yet. Committing it as literal and bolding it on
735
+ the replay is exactly the divergence the hold exists to prevent.
736
+ """
737
+ if not _em_edge(text, index, opening=True):
738
+ return None, False
739
+ at = index + 1
740
+ while True:
741
+ close = text.find("*", at)
742
+ if close == -1:
743
+ return None, True
744
+ if close > index + 1 and _em_edge(text, close, opening=False):
745
+ if close - index > HOLD_LIMIT:
746
+ return None, False
747
+ return (text[index + 1:close], EM, index, close + 1), False
748
+ at = close + 1
749
+
750
+
751
+ def clickable(url) -> bool:
752
+ """May this URL be armed for a click (batch 58)?
753
+
754
+ ASCII ONLY, and that is the security half rather than a tidiness
755
+ one. The design rule is that the visible text IS the target, so
756
+ nothing can hide a destination behind a friendly label -- and a
757
+ homograph is precisely the case where the visible form lies. A URL
758
+ with non-ASCII in it still renders and is still copied by `/copy`; it
759
+ simply is not armed.
760
+
761
+ http and https only: the click ends in the platform's URL handler,
762
+ and a `file://` or a registered custom scheme is not something a
763
+ model gets to reach through a transcript.
764
+
765
+ NO USERINFO, which batch 65 added after measuring that the rule
766
+ above had a hole exactly its own shape.
767
+ `https://accounts.google.com@phish.example/x` is ASCII and https,
768
+ so it was armed -- and it opens phish.example. The visible text IS
769
+ the target, so the letter of the rule held; what a reader takes for
770
+ the destination is the userinfo, and the host is somewhere else.
771
+ That is the same attack `[label](url)` was refused for, arriving
772
+ through the URL's own syntax rather than through a label.
773
+
774
+ NOR A URL THE HARNESS REWROTE, which is the same argument one step
775
+ further and was found by a test rather than by reasoning. The
776
+ userinfo rule alone catches
777
+ `https://user:[REDACTED]@host/...`, and nothing caught
778
+ `https://example.com/x?api_key=[REDACTED]` -- ASCII, https, no
779
+ userinfo, and a perfectly real address that is simply not the one
780
+ that was fetched. Clicking it would send the literal string
781
+ `[REDACTED]` to a host as an api_key.
782
+
783
+ A redacted URL still RENDERS and is still copied, exactly as a
784
+ homograph is: the rule is about what a click may do, never about
785
+ hiding text from the reader.
786
+ """
787
+ if not isinstance(url, str) or not url.isascii():
788
+ return False
789
+ if any(ord(char) < 32 or ord(char) == 127 for char in url):
790
+ return False
791
+ if not (url.startswith("http://") or url.startswith("https://")):
792
+ return False
793
+ if REDACTED_MARK in url:
794
+ return False
795
+ return "@" not in _authority(url)
796
+
797
+
798
+ def _authority(url: str) -> str:
799
+ """Everything between `scheme://` and the path, query or fragment.
800
+
801
+ Written out rather than `urllib.parse`d, for this module's rule: it
802
+ imports `re` and nothing else, and a parser that normalises would
803
+ answer about a URL slightly different from the one on screen --
804
+ which is the exact gap the whole link design exists to close.
805
+ """
806
+ rest = url.split("://", 1)[-1]
807
+ for stop in "/?#":
808
+ rest = rest.split(stop, 1)[0]
809
+ return rest
810
+
811
+
812
+ def _trim_url(url: str) -> str:
813
+ """A detected URL without the prose punctuation stuck to its end.
814
+
815
+ Balanced brackets survive -- `https://x/a_(b)` keeps its own -- and
816
+ an unbalanced one does not, which is what makes the URL inside
817
+ `[docs](https://x/a)` come out as the URL rather than with the
818
+ markdown link's closing parenthesis welded on.
819
+ """
820
+ while url:
821
+ last = url[-1]
822
+ if last in _URL_TRIM:
823
+ url = url[:-1]
824
+ continue
825
+ opener = _URL_PAIRS.get(last)
826
+ if opener is not None and url.count(last) > url.count(opener):
827
+ url = url[:-1]
828
+ continue
829
+ break
830
+ return url
831
+
832
+
833
+ def _url(text: str, index: int):
834
+ """`(span, opened)` for a bare URL starting at `index`.
835
+
836
+ Never opens: a URL has no closing delimiter, so there is nothing to
837
+ wait for and nothing to hold. A URL that cannot be armed returns no
838
+ span at all and falls through to the literal path, which is how a
839
+ non-ASCII one stays visible without becoming clickable.
840
+ """
841
+ if index and text[index - 1].isalnum():
842
+ return None, False
843
+ match = _URL.match(text, index)
844
+ if match is None:
845
+ return None, False
846
+ url = _trim_url(match.group(0))
847
+ if not clickable(url):
848
+ return None, False
849
+ return (url, LINK, index, index + len(url)), False
850
+
851
+
852
+ def _scan(text: str, *, block: bool = True):
853
+ """`(spans, first unclosed mark)` for one line, in ONE walk.
854
+
855
+ The two answers come from the same pass because they are the same
856
+ reading: a mark is a span when it closes on this line and an open mark
857
+ when it does not, and two tokenisers asked to agree about that would
858
+ agree right up until they did not. `inline_spans` wants the first half,
859
+ `safe_commit_limit` the second.
860
+
861
+ The source extents are what `width_split` cuts on: a cut INSIDE a mark
862
+ would leave the delimiters unbalanced on both sides of the break, so
863
+ each half would re-parse as literal asterisks while the replay drew one
864
+ styled span. Same divergence the cap exists to prevent, one layer down.
865
+
866
+ A LONE TRAILING `*` counts as open. It is not a mark in this grammar and
867
+ renders as itself, but mid-stream it is the first half of a `**` that
868
+ has not finished arriving -- and the hard-cut branch of `width_split` is
869
+ able to sever exactly there, on a row with no spaces in it.
870
+
871
+ An INDENTED line is one unmarked span and nothing else (batch 58) --
872
+ `verbatim`'s rule, applied HERE so that every caller inherits it:
873
+ `inline_spans` draws it as written, `display_len` measures the
874
+ characters it will really draw, and `width_split` cuts on them.
875
+ """
876
+ if block and verbatim(text):
877
+ return ([(text, "", 0, len(text))] if text else []), None
878
+
879
+ out, buffer, start, index = [], [], 0, 0
880
+ open_at = None
881
+ length = len(text)
882
+
883
+ def flush(end):
884
+ if buffer:
885
+ out.append(("".join(buffer), "", start, end))
886
+ buffer.clear()
887
+
888
+ while index < length:
889
+ char = text[index]
890
+ span = None
891
+ opened = False
892
+ if char == "h":
893
+ span, opened = _url(text, index)
894
+ elif char == "`":
895
+ span, opened = _paired(text, index, "`", CODE)
896
+ elif text.startswith("~~", index):
897
+ span, opened = _paired(text, index, "~~", STRIKE)
898
+ elif text.startswith("**", index):
899
+ span, opened = _paired(text, index, "**", STRONG)
900
+ elif char == "*":
901
+ span, opened = _em(text, index)
902
+ if span is not None:
903
+ flush(index)
904
+ out.append(span)
905
+ index = span[3]
906
+ continue
907
+ if opened and open_at is None:
908
+ open_at = index
909
+ if not buffer:
910
+ start = index
911
+ buffer.append(char)
912
+ index += 1
913
+
914
+ # A trailing RUN of `*`, `~` or `` ` `` that survived as literal text is
915
+ # the first half of a mark that has not finished arriving -- `**`, or
916
+ # the ``` of a fence. None is a mark in this grammar and all render as
917
+ # themselves, but the hard-cut branch of `width_split` can sever exactly
918
+ # there, on a row with no spaces in it, and a fence severed at its second
919
+ # backtick opens a block the replay never sees.
920
+ if buffer:
921
+ tail = "".join(buffer)
922
+ run = len(tail) - len(tail.rstrip("`*~"))
923
+ if run and (open_at is None or length - run < open_at):
924
+ open_at = length - run
925
+ flush(length)
926
+ return out, open_at
927
+
928
+
929
+ def display_len(text: str) -> int:
930
+ """Cells this line will occupy once its marks are rendered.
931
+
932
+ Multi-line input is measured line by line and summed without the
933
+ newlines, which is the only sensible reading for a caller asking "how
934
+ wide is this" -- and the only caller that matters asks it about a single
935
+ line, because the newline rule has already taken everything above it.
936
+ """
937
+ total = 0
938
+ for line in text.split("\n"):
939
+ # Through `inline_spans`, not `_spans`: a heading's `#` is dropped
940
+ # before it is drawn, so a measurement that counted it would report
941
+ # a width the terminal never sees.
942
+ total += sum(cell_len(span) for span, _role in inline_spans(line))
943
+ return total
944
+
945
+
946
+ def _cut_points(spans):
947
+ """`(source index, cells drawn before it)` for every position a row may
948
+ break at, in increasing order.
949
+
950
+ A mark offers its two ends and nothing between them: a cut INSIDE one
951
+ would leave the delimiters unbalanced on both sides of the break.
952
+ Unmarked text offers every position inside it. One reading, shared by
953
+ the streaming cut and the wrap, so the two cannot disagree about where
954
+ a row ends.
955
+ """
956
+ points, cells = [], 0
957
+ for text, role, source, end in spans:
958
+ if role:
959
+ points.append((source, cells))
960
+ cells += cell_len(text)
961
+ points.append((end, cells))
962
+ continue
963
+ for offset, char in enumerate(text):
964
+ points.append((source + offset, cells))
965
+ cells += cell_len(char)
966
+ points.append((end, cells))
967
+ return points
968
+
969
+
970
+ def width_split(line: str, width: int, *, block: bool = True):
971
+ """`(commit now, keep buffered)` for one unterminated line.
972
+
973
+ `Transcript._split_committable`'s width rule, measured in RENDERED cells
974
+ and cut at positions that do not fall inside a mark. The shape is
975
+ deliberately the source rule's:
976
+
977
+ - shorter than a row: nothing commits;
978
+ - otherwise cut after the last space that still fits;
979
+ - no space fits: cut at the width, which is what Rich does with a word
980
+ too long for the line (a URL, the case this branch exists for).
981
+
982
+ The one addition is that every candidate position is a span boundary or
983
+ a point inside an unmarked span. A run with no such position -- a single
984
+ `**...**` longer than a row with no spaces in it -- commits nothing and
985
+ waits for the newline, which is the cap's answer applied to the one
986
+ shape the cap itself cannot see.
987
+
988
+ `block` is `inline_spans`' -- False when the text is the tail of a line
989
+ already partly drawn, where the indent rule must not fire because those
990
+ leading spaces are not an indent.
991
+ """
992
+ spans = _spans(line, block=block)
993
+ if not spans:
994
+ return "", line
995
+ if sum(cell_len(text) for text, _r, _s, _e in spans) <= width:
996
+ return "", line
997
+
998
+ points = _cut_points(spans)
999
+ space = None
1000
+ for index, before in points:
1001
+ if before > width:
1002
+ break
1003
+ if index < len(line) and line[index] == " ":
1004
+ space = index
1005
+ if space is not None:
1006
+ return line[:space + 1], line[space + 1:]
1007
+
1008
+ hard = 0
1009
+ for index, before in points:
1010
+ if before > width:
1011
+ break
1012
+ hard = index
1013
+ if hard <= 0:
1014
+ return "", line
1015
+ return line[:hard], line[hard:]
1016
+
1017
+
1018
+ def plain_split(line: str, width: int):
1019
+ """`(commit now, keep buffered)` for thinking text, measured in cells.
1020
+
1021
+ `Transcript._split_committable`'s `marks=False` branch, which is the
1022
+ thinking path. Reasoning renders as prose, marks and all, so parsing
1023
+ marks here would describe a rendering that does not happen -- but the
1024
+ old branch measured `len()` characters, and a double-width glyph is
1025
+ one character and two cells, so thinking held a row Rich had already
1026
+ wrapped. Same shape as `width_split` (last space that fits, else a
1027
+ hard cut at the width Rich would use), over plain characters rather
1028
+ than mark spans. A character is atomic: the cut never lands inside
1029
+ one, so a glyph wider than the row survives whole for Rich to wrap.
1030
+
1031
+ The rule itself is `_plain_cut`, shared with `plain_wrap`, so the
1032
+ streamed commit and the replayed wrap cannot disagree about a row.
1033
+ """
1034
+ cut = _plain_cut(line, 0, width)
1035
+ if cut is None:
1036
+ return "", line
1037
+ return line[:cut], line[cut:]
1038
+
1039
+
1040
+ def _plain_cut(line: str, start: int, width: int):
1041
+ """Where `plain_split`'s rule ends the row that begins at `start`, or
1042
+ None when the rest of the line fits in one row or cannot be cut.
1043
+
1044
+ Last space that still fits, else a hard cut at the width -- and never
1045
+ at `start` itself, which is how a glyph wider than the row survives
1046
+ whole. It scans about one row past `start`, never the whole
1047
+ remainder, which is what lets `plain_wrap` walk a long line once.
1048
+ """
1049
+ cells, space_at = 0, -1
1050
+ for index in range(start, len(line)):
1051
+ char = line[index]
1052
+ if char == " " and cells <= width:
1053
+ space_at = index
1054
+ cells += cell_len(char)
1055
+ if cells > width:
1056
+ break
1057
+ else:
1058
+ return None
1059
+ if space_at > start:
1060
+ return space_at + 1
1061
+ cells = 0
1062
+ for index in range(start, len(line)):
1063
+ char_cells = cell_len(line[index])
1064
+ if cells + char_cells > width:
1065
+ return index if index > start else None
1066
+ cells += char_cells
1067
+ return None
1068
+
1069
+
1070
+ def plain_wrap(line: str, width: int) -> list[str]:
1071
+ """The rows `line` occupies when drawn at `width` cells, by
1072
+ `plain_split`'s rule -- the thinking span's wrap.
1073
+
1074
+ `plain_split` applied until it commits nothing, but in one walk over
1075
+ the line: calling it in a loop re-measures the whole remainder for
1076
+ every row, which is quadratic on a long paragraph and paid again on
1077
+ every replay (`wrap_display`'s reason, one grammar over). The last row
1078
+ is what that rule would have kept buffered, so a line that fits comes
1079
+ back as itself and an empty line as one empty row.
1080
+
1081
+ A cut at the very END of the line -- a trailing space that overflows
1082
+ the row by its own cell -- closes the last row rather than opening an
1083
+ empty one. `plain_split` hands that empty remainder back, and Rich
1084
+ folds the space into the row it ends, so drawing the remainder put a
1085
+ bar with nothing beside it under every chunk `thinking_delta` commits
1086
+ that way (measured: three extra rows in a streamed run of wide glyphs).
1087
+ """
1088
+ if width <= 0:
1089
+ return [line]
1090
+ rows, start = [], 0
1091
+ while True:
1092
+ cut = _plain_cut(line, start, width)
1093
+ if cut is None:
1094
+ break
1095
+ rows.append(line[start:cut])
1096
+ start = cut
1097
+ if start < len(line) or not rows:
1098
+ rows.append(line[start:])
1099
+ return rows
1100
+
1101
+
1102
+ def wrap_display(line: str, width: int, *, block: bool = False):
1103
+ """The rows `line` occupies when drawn at `width` cells.
1104
+
1105
+ `width_split`'s rule applied until the text runs out, but in ONE pass
1106
+ over ONE points list rather than by calling it in a loop -- which
1107
+ would rescan from the start of the remainder each time and go
1108
+ quadratic on a long line, once per render and again on every `/theme`.
1109
+
1110
+ `block` defaults to False because the only caller wraps a list item's
1111
+ BODY, which is a fragment of its line: the marker has already been
1112
+ taken off the front, so nothing about where the line started applies
1113
+ to it any more.
1114
+
1115
+ A span wider than the row survives whole -- a URL, mostly -- and Rich
1116
+ soft-wraps it afterwards. That is the same answer `width_split` gives
1117
+ the same shape, and it is identical on both paths because both reach
1118
+ it through here.
1119
+ """
1120
+ if width <= 0 or not line:
1121
+ return [line]
1122
+ points = _cut_points(_spans(line, block=block))
1123
+ if not points:
1124
+ return [line]
1125
+
1126
+ rows, start, base, space = [], 0, 0, None
1127
+ for index, cells in points:
1128
+ if cells - base > width:
1129
+ cut = space if space is not None else None
1130
+ if cut is None and index > start:
1131
+ cut = (index, cells)
1132
+ if cut is not None:
1133
+ rows.append(line[start:cut[0]])
1134
+ start, base, space = cut[0], cut[1], None
1135
+ if index < len(line) and line[index] == " ":
1136
+ space = (index + 1, cells + 1)
1137
+ rows.append(line[start:])
1138
+ return rows
1139
+
1140
+
1141
+ # ---------------------------------------------------------------------------
1142
+ # ---- The commit cap -------------------------------------------------------
1143
+ # ---------------------------------------------------------------------------
1144
+
1145
+ def _opens_line_construct(line: str) -> bool:
1146
+ """Does this line start a heading or a list item, LENGTH ASIDE?
1147
+
1148
+ The cap's half of `heading` and `list_item`, sharing their patterns
1149
+ but not their HOLD_LIMIT guard -- because the limit is the thing this
1150
+ function's caller ENFORCES. The renderer asks "is this one", and past
1151
+ the limit the answer is no; the cap asks "should I still be waiting",
1152
+ and it has to hold first in order to give up afterwards.
1153
+ """
1154
+ return not verbatim(line) and bool(_HEADING.match(line)
1155
+ or _LIST.match(line))
1156
+
1157
+
1158
+ def commit_span(committed: str, pending: str):
1159
+ """`(limit, forced)` -- how much may be drawn, and whether the cap gave up.
1160
+
1161
+ `safe_commit_limit` below is this function's first value and carries
1162
+ the documentation for the cap itself. `forced` is true when a
1163
+ LINE-SCOPED hold was abandoned because it passed HOLD_LIMIT, and it
1164
+ means one thing to the caller: draw this chunk as though it were the
1165
+ middle of a line, because that is what it is. The line it opened is
1166
+ no longer a heading or a list item on either path -- `heading` and
1167
+ `list_item` refuse it at the same limit -- so the released rows and
1168
+ the replayed ones are the same rows.
1169
+
1170
+ A FENCE and a TABLE are never forced. Both must be drawn whole to be
1171
+ drawn correctly: half a fence inverts the fence parity of everything
1172
+ after it, and half a table is two stacked grids. Their holds end at
1173
+ the closing fence, at the first line that is not a row, or at
1174
+ `flush_stream` when the turn does. That is a narrower guarantee than
1175
+ the line-scoped constructs get, and it is the honest one.
1176
+ """
1177
+ block_at = 0 if _fence_line_count(committed) % 2 == 1 else None
1178
+ fence_open = block_at is not None
1179
+ line_at = None
1180
+ lines = pending.splitlines(keepends=True)
1181
+ in_table = False
1182
+ offset = 0
1183
+
1184
+ for index, raw in enumerate(lines):
1185
+ start = offset
1186
+ offset += len(raw)
1187
+ complete = raw.endswith("\n")
1188
+ line = _strip_eol(raw)
1189
+
1190
+ if fence_open:
1191
+ if _is_fence_line(line):
1192
+ fence_open = False
1193
+ # ONLY once that line has ended. §38's own pin is this
1194
+ # case: a block whose closing fence has no newline after
1195
+ # it yet puts the last newline INSIDE the block, so the
1196
+ # otherwise-committable prefix would draw an unterminated
1197
+ # opener as plain text. The fence is closed for the scan's
1198
+ # purposes; the CAP stays where the opener put it.
1199
+ if complete:
1200
+ block_at = None
1201
+ continue
1202
+
1203
+ # A table ends at the first line that is not a row, and a fence
1204
+ # opener is one -- so this is settled BEFORE the fence branch. The
1205
+ # other order clears the table's hold by never reaching it, and
1206
+ # then the fence's own hold reports a later offset than the table
1207
+ # it silently swallowed.
1208
+ if in_table:
1209
+ if _is_row(line):
1210
+ continue
1211
+ in_table, block_at = False, None
1212
+
1213
+ if _is_fence_line(line):
1214
+ fence_open, block_at = True, _hold(block_at, start)
1215
+ continue
1216
+
1217
+ if _is_row(line):
1218
+ verdict = _table_verdict(lines, index, line)
1219
+ if verdict is None: # cannot tell yet
1220
+ block_at = _hold(block_at, start)
1221
+ continue
1222
+ if verdict:
1223
+ in_table, block_at = True, _hold(block_at, start)
1224
+ continue
1225
+ # settled as prose: fall through and treat it as a normal line
1226
+
1227
+ if not complete:
1228
+ if verbatim(line):
1229
+ continue
1230
+ if _opens_line_construct(line):
1231
+ line_at = _hold(line_at, start)
1232
+ continue
1233
+ mark = _first_open_mark(line)
1234
+ if mark is not None:
1235
+ line_at = _hold(line_at, start + mark)
1236
+
1237
+ # A block hold always sits at or before a line hold -- the branches
1238
+ # that set one `continue` past the branch that sets the other -- so
1239
+ # it wins outright rather than by comparison.
1240
+ if block_at is not None:
1241
+ return block_at, False
1242
+ if line_at is None:
1243
+ return len(pending), False
1244
+ if len(pending) - line_at > HOLD_LIMIT:
1245
+ return len(pending), True
1246
+ return line_at, False
1247
+
1248
+
1249
+ def safe_commit_limit(committed: str, pending: str) -> int:
1250
+ """How much of `pending` may be drawn now, as an index into it.
1251
+
1252
+ §38 held a streamed chunk while a ``` fence was open, for a reason that
1253
+ was never only about fences: RichLog appends and cannot rewrite a drawn
1254
+ row, so a construct committed in halves renders as its own source and
1255
+ can never be put right. This is that rule as a CAP over four constructs
1256
+ rather than a single rejection.
1257
+
1258
+ | construct | held from | released by |
1259
+ |----------------------|-------------------|------------------------------|
1260
+ | open ``` fence | the opening fence | the closing fence |
1261
+ | table | the header row | a non-row line, or the flush |
1262
+ | heading | the `#` | that line's newline |
1263
+ | list item | the marker | that line's newline |
1264
+ | unclosed `**` / `` ` ``| the mark | its closer, or the newline |
1265
+
1266
+ A LIST ITEM joined that table in batch 58, for the reason the others
1267
+ are in it: its wrapped rows are padded to the marker's own text
1268
+ column, and a fragment committed without the marker can never be
1269
+ indented afterwards.
1270
+
1271
+ The bottom three are ALSO bounded by HOLD_LIMIT. See `commit_span`,
1272
+ which is this function plus the flag saying the bound was reached.
1273
+
1274
+ A CAP rather than a rejection is strictly better than what it replaces:
1275
+ a paragraph sitting in the same buffer as a fence used to wait for the
1276
+ fence to close, and now streams. It is also backward compatible with
1277
+ §38's two pins, where the fence starts at index 0 and the cap is
1278
+ therefore 0.
1279
+
1280
+ A TABLE HAS NO TERMINATOR, so one is held until the model writes a line
1281
+ that is not a row, or until the turn ends -- the same trade the fence
1282
+ hold already makes, and the reason `flush_stream` renders what it has.
1283
+
1284
+ Only the LAST line can be capped for a heading or an unclosed mark: on a
1285
+ line that has ended, an unmatched `**` is literal text and renders that
1286
+ way on both paths, so holding it would be waiting for something that
1287
+ already happened.
1288
+ """
1289
+ return commit_span(committed, pending)[0]
1290
+
1291
+
1292
+ def _hold(open_at, start):
1293
+ """The EARLIER of two unresolved offsets.
1294
+
1295
+ A cap is one number, and a construct that opens later cannot license
1296
+ drawing through one that is still open before it. Taking the minimum
1297
+ rather than assigning is what a pipe line under a pipe line found: both
1298
+ are undecidable headers, and the second one was reporting its own
1299
+ offset as the limit -- so the first line committed as prose and the
1300
+ delimiter that arrived next had nowhere to attach.
1301
+ """
1302
+ return start if open_at is None else min(open_at, start)
1303
+
1304
+
1305
+ def _table_verdict(lines, index, header):
1306
+ """True (a table starts here), False (prose), or None (undecidable yet).
1307
+
1308
+ The undecidable case is the one worth naming: a line carrying a pipe is
1309
+ only a header once the line UNDER it has been seen, so it has to be held
1310
+ until then. `_delimiter_possible` is what keeps that hold to a few
1311
+ tokens instead of a paragraph -- the first letter of an ordinary
1312
+ sentence settles it as prose.
1313
+ """
1314
+ if index + 1 >= len(lines):
1315
+ return None
1316
+ following = lines[index + 1]
1317
+ if following.endswith("\n"):
1318
+ return _delimiter_aligns(_strip_eol(following),
1319
+ len(_cells(header))) is not None
1320
+ if _delimiter_possible(following):
1321
+ return None
1322
+ return False
1323
+
1324
+
1325
+ def _first_open_mark(line: str):
1326
+ """Index of the first mark on this line that has not been closed, or
1327
+ None. Read off the one tokeniser rather than counted separately, so
1328
+ there is no second reading to agree with it until it does not."""
1329
+ return _scan(line)[1]