@yahyahammad/venastine 0.1.0 → 0.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +19 -14
- package/CONFIG_ARCHITECTURE.md +2002 -0
- package/NOTICE +1 -1
- package/README.md +249 -93
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/plan.md +23 -5
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +261 -195
- package/agents/subagent_tool.py +280 -23
- package/agents/tui_commands.py +31 -21
- package/bin/venastine.mjs +528 -382
- package/bin/venastine.py +71 -0
- package/cli_commands.py +835 -0
- package/config.py +95 -1198
- package/config.yaml +1090 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +966 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -0
- package/core/ansi.py +452 -0
- package/core/approval.py +27 -5
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +1249 -82
- package/core/commands.py +454 -0
- package/core/compaction.py +1735 -959
- package/core/config_loader.py +516 -97
- package/core/connect.py +686 -0
- package/core/events.py +68 -6
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +87 -4
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1353
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +23 -9
- package/core/provider_errors.py +234 -0
- package/core/reasoning/confidence_scoring.py +70 -54
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +232 -53
- package/core/reasoning/output_writer.py +90 -31
- package/core/reasoning/payload_validation.py +133 -7
- package/core/reasoning/pipeline_storage.py +427 -361
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +38 -14
- package/core/reasoning/source_corpus.py +110 -4
- package/core/replay.py +315 -167
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +308 -180
- package/database.py +82 -1
- package/{PRIVACY.md → docs/PRIVACY.md} +19 -7
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +118 -77
- package/json_store.py +236 -16
- package/logging_setup.py +121 -11
- package/main.py +1672 -198
- package/mcp_client/client.py +257 -23
- package/mcp_client/config.py +484 -379
- package/mcp_client/registration.py +163 -2
- package/memories/manager.py +15 -0
- package/memories/tui_commands.py +21 -14
- package/package.json +303 -73
- package/project_init/config_files.py +4 -0
- package/project_init/generator.py +201 -37
- package/project_init/manifest.py +418 -406
- package/project_init/tui_commands.py +7 -4
- package/prompts/revalidate.md +1 -1
- package/prompts/revise.md +17 -15
- package/prompts/system_prompts.py +64 -10
- package/providers.json.example +94 -92
- package/pyproject.toml +149 -100
- package/requirements.txt +252 -29
- package/safety/policy_enforcement.py +1082 -682
- package/security/askpass.py +419 -0
- package/security/capability.py +37 -7
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +54 -7
- package/security/posture.py +124 -12
- package/security/protected_paths.py +198 -6
- package/security/sandbox.py +3262 -139
- package/security/secrets.py +855 -0
- package/skills/tui_commands.py +3 -4
- package/storage.py +1338 -1005
- package/tools/base.py +465 -305
- package/tools/builtin/_math_common.py +306 -16
- package/tools/builtin/_net_common.py +267 -61
- package/tools/builtin/arxiv.py +681 -223
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +239 -21
- package/tools/builtin/file_ops.py +341 -33
- package/tools/builtin/geometry.py +184 -148
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +10 -0
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +46 -12
- package/tools/builtin/probability_stats.py +224 -228
- package/tools/builtin/project_docs.py +80 -21
- package/tools/builtin/remember.py +21 -4
- package/tools/builtin/shell.py +586 -67
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +5 -3
- package/tools/builtin/web_search.py +329 -151
- package/tools/context.py +23 -0
- package/tools/isolation.py +387 -314
- package/tools/registry.py +374 -29
- package/tui/app.py +5281 -397
- package/tui/app.tcss +290 -9
- package/tui/commands.py +26 -61
- package/tui/diffs.py +286 -265
- package/tui/history.py +128 -0
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -0
- package/tui/meters.py +364 -0
- package/tui/screens.py +346 -92
- package/tui/themes.py +599 -366
- package/tui/widgets.py +2673 -116
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/tui/markdown.py
ADDED
|
@@ -0,0 +1,1329 @@
|
|
|
1
|
+
"""
|
|
2
|
+
tui/markdown.py
|
|
3
|
+
|
|
4
|
+
What a markdown answer is made of, as data (batch 53).
|
|
5
|
+
|
|
6
|
+
The transcript used to render exactly one construct -- a ``` fence, through
|
|
7
|
+
`rich.syntax.Syntax` -- and hand everything else to `Text` unchanged. So a
|
|
8
|
+
table arrived as its own pipes, and `_split_fences`' docstring said why:
|
|
9
|
+
"deliberately small: this is a renderer, not a markdown parser." This module
|
|
10
|
+
is the parser that sentence declined to write, kept to the constructs a
|
|
11
|
+
reader actually loses by not having: a TABLE, a HEADING, the inline marks
|
|
12
|
+
(`**strong**`, `` `code` ``, and batch 58's `*emphasis*` and `~~strike~~`), a
|
|
13
|
+
LIST ITEM, and a bare URL.
|
|
14
|
+
|
|
15
|
+
CURATED rather than complete, and the omissions are decisions with reasons
|
|
16
|
+
recorded in AGENTS.md: no block quotes (a bar down the left collides with the
|
|
17
|
+
thinking span's own bar), no horizontal rules, no heading LEVELS, no setext
|
|
18
|
+
headings, nothing inside a reasoning span.
|
|
19
|
+
|
|
20
|
+
PURE. No I/O, no `core/` import, no harness state -- `tui/diffs.py`'s rule,
|
|
21
|
+
because `tui/widgets.py` imports this the same way. Parsing lives here;
|
|
22
|
+
building the Rich renderables stays in the widget, so a replay under a new
|
|
23
|
+
theme re-derives the painting rather than replaying something decorated once.
|
|
24
|
+
|
|
25
|
+
INSIDE the `tui` package on purpose. A root-level `markdown.py` would sit on
|
|
26
|
+
`sys.path` beside the interpreter's own imports -- the `logging.py` incident,
|
|
27
|
+
and the reason `mcp_client/` is not called `mcp/`. Nothing here shadows
|
|
28
|
+
`rich.markdown`, which is imported absolutely and by nobody in this project.
|
|
29
|
+
|
|
30
|
+
WHAT THIS OWNS, and the last one is the one that is easy to miss.
|
|
31
|
+
|
|
32
|
+
- `split_blocks` -- plain text, fenced code, tables.
|
|
33
|
+
- `verbatim` / `heading` / `list_item` -- what one LINE is.
|
|
34
|
+
- `inline_spans` -- one line as `(text, role)` pairs.
|
|
35
|
+
- `wrap_display` -- the rows a list item's body will occupy.
|
|
36
|
+
- `commit_span` -- how far a streamed answer may be drawn RIGHT NOW, and
|
|
37
|
+
whether the cap gave up. `safe_commit_limit` is its first value.
|
|
38
|
+
|
|
39
|
+
That third one is the whole reason the first two are safe. `Transcript`
|
|
40
|
+
commits a streamed answer at boundaries it computes itself, and RichLog
|
|
41
|
+
cannot rewrite a row it has drawn -- so a construct must never be committed
|
|
42
|
+
in halves. §38 shipped one instance of that rule (an open fence is held);
|
|
43
|
+
this generalises it to a CAP: the offset past which committing would split
|
|
44
|
+
something. FIVE constructs answer to it, listed at the function, and batch 58
|
|
45
|
+
put a ceiling on the three of them that are waiting for a newline no model is
|
|
46
|
+
obliged to send.
|
|
47
|
+
|
|
48
|
+
MEASUREMENT IS THE OTHER HALF, and the two are useless apart. `**bold**`
|
|
49
|
+
renders four cells shorter than its source, so a cut taken in SOURCE columns
|
|
50
|
+
lands where Rich would not have wrapped -- and the streamed rows then differ
|
|
51
|
+
from the ones `rerender()` draws from the same text, which is the divergence
|
|
52
|
+
`TestAStreamedAnswerRendersLikeAWrittenOne` exists to catch. `display_len`
|
|
53
|
+
and `width_split` measure what will actually be on screen; the cap is what
|
|
54
|
+
makes that measurable, because no cut is ever offered while a mark is still
|
|
55
|
+
open and its rendered width therefore still unknown.
|
|
56
|
+
|
|
57
|
+
Cells rather than characters (`rich.cells.cell_len`): Rich wraps by cells,
|
|
58
|
+
so this is the same question Rich will ask, and a double-width glyph is
|
|
59
|
+
counted the way the terminal will draw it.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
import re
|
|
63
|
+
|
|
64
|
+
from rich.cells import cell_len
|
|
65
|
+
|
|
66
|
+
#: The fence delimiter, spelled once. `Transcript._fence_is_open` counts
|
|
67
|
+
#: these and this module splits on them, and two spellings of one token is
|
|
68
|
+
#: the shape this project keeps finding drift in.
|
|
69
|
+
FENCE = "```"
|
|
70
|
+
|
|
71
|
+
#: Roles for the marks, resolved against the active theme by
|
|
72
|
+
#: `tui/themes.role_styles`. Strings rather than an enum for the reason the
|
|
73
|
+
#: transcript's own roles are strings: they are palette keys, and the
|
|
74
|
+
#: palette is a dict.
|
|
75
|
+
HEADING = "md_heading"
|
|
76
|
+
STRONG = "md_strong"
|
|
77
|
+
CODE = "md_code"
|
|
78
|
+
#: Batch 58. `LINK` is the one role whose span TEXT carries information: a
|
|
79
|
+
#: link span is always the URL itself, so the renderer needs no second
|
|
80
|
+
#: channel to know what a click should open. That is the security rule
|
|
81
|
+
#: spelled as a data shape -- there is no label to hide a target behind.
|
|
82
|
+
EM = "md_em"
|
|
83
|
+
STRIKE = "md_strike"
|
|
84
|
+
LINK = "md_link"
|
|
85
|
+
BULLET = "md_bullet"
|
|
86
|
+
|
|
87
|
+
#: The ceiling on a hold (batch 58). `safe_commit_limit` holds a construct
|
|
88
|
+
#: rather than commit it in halves, and three of those holds are waiting on
|
|
89
|
+
#: a newline that a model is under no obligation to send. Past this many
|
|
90
|
+
#: characters the construct stops being one -- on BOTH paths, which is what
|
|
91
|
+
#: keeps the streamed rows equal to the replayed ones -- and the stream is
|
|
92
|
+
#: released. A fence and a table are exempt and say why at the function.
|
|
93
|
+
#:
|
|
94
|
+
#: Characters rather than rows, deliberately: a rule measured in rows would
|
|
95
|
+
#: depend on the terminal, so a resize or a `/theme` could re-classify a
|
|
96
|
+
#: line and `rerender()` would draw something the live path never drew.
|
|
97
|
+
HOLD_LIMIT = 1000
|
|
98
|
+
|
|
99
|
+
#: Indented at or past this, a line is drawn exactly as it was written.
|
|
100
|
+
#: Markdown's own code-block indent, used here only as a NO-PARSE zone --
|
|
101
|
+
#: this module does not render indented code as a block, it declines to
|
|
102
|
+
#: read anything into one. Fences are exempt; see TECHNICAL_DEBT.md.
|
|
103
|
+
CODE_INDENT = 4
|
|
104
|
+
|
|
105
|
+
#: ATX headings only, and the space is REQUIRED -- `#hashtag` is not a
|
|
106
|
+
#: heading, and `###` alone is not one either. Closing hashes are left
|
|
107
|
+
#: alone: GFM strips them, and a rule that eats trailing `#` characters
|
|
108
|
+
#: would silently edit a line that merely ends in one.
|
|
109
|
+
_HEADING = re.compile(r"^(#{1,6})\s+")
|
|
110
|
+
|
|
111
|
+
#: One cell of a GFM delimiter row: dashes, optionally anchored by a colon
|
|
112
|
+
#: at either end to set the column's alignment.
|
|
113
|
+
_DELIMITER_CELL = re.compile(r"^:?-+:?$")
|
|
114
|
+
|
|
115
|
+
#: Everything a delimiter row may be built from. Used to decide, from a
|
|
116
|
+
#: PARTIAL next line, whether a table is still possible -- one character
|
|
117
|
+
#: outside this set settles it, which is what keeps a paragraph following
|
|
118
|
+
#: a pipe in prose from being held until its own newline arrives.
|
|
119
|
+
_DELIMITER_CHARS = set("|-: \t")
|
|
120
|
+
|
|
121
|
+
#: A list item's opener: bullet or ordered, then ONE to FOUR spaces. The
|
|
122
|
+
#: upper bound is GFM's -- past four the content is an indented block
|
|
123
|
+
#: rather than the item's text -- and it also stops a stray `- ` in a
|
|
124
|
+
#: column of ASCII art from claiming a hanging indent it cannot fill.
|
|
125
|
+
#: The leading run is unbounded here because `verbatim()` is what rules
|
|
126
|
+
#: out an indented line, so CODE_INDENT stays the single spelling of that.
|
|
127
|
+
_LIST = re.compile(r"^( *)([-*+]|\d{1,9}[.)])( {1,4})(?! )(.*)$")
|
|
128
|
+
|
|
129
|
+
#: A bare URL, and the only thing in this grammar that becomes clickable.
|
|
130
|
+
#: Ends at whitespace or at a character that cannot appear in one without
|
|
131
|
+
#: being escaped -- the brackets of a markdown link are NOT excluded, so
|
|
132
|
+
#: `[docs](https://x/a)` yields the URL and leaves the label as prose.
|
|
133
|
+
_URL = re.compile(r"https?://[^\s<>\"'`\\]+")
|
|
134
|
+
|
|
135
|
+
#: Trailing characters trimmed off a detected URL: sentence punctuation
|
|
136
|
+
#: and the marks that sit beside one in prose. A closing bracket is
|
|
137
|
+
#: handled separately, by BALANCE, so `https://x/a_(b)` keeps its own.
|
|
138
|
+
_URL_TRIM = ".,;:!?*_~'\""
|
|
139
|
+
|
|
140
|
+
#: Bracket pairs whose closer is trimmed only when it was never opened.
|
|
141
|
+
_URL_PAIRS = {")": "(", "]": "[", "}": "{"}
|
|
142
|
+
|
|
143
|
+
#: What `safety.policy_enforcement` leaves where a secret was, and the
|
|
144
|
+
#: one thing this module knows about that one. Its own copy rather than
|
|
145
|
+
#: an import, because this file is pure -- it imports `re` and nothing
|
|
146
|
+
#: else -- and reaching into policy for a nine-character string would
|
|
147
|
+
#: cost that for no gain. `tests/test_markdown_render.py` holds the two
|
|
148
|
+
#: against each other, so the duplication is checked rather than
|
|
149
|
+
#: trusted.
|
|
150
|
+
REDACTED_MARK = "[REDACTED]"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# ---------------------------------------------------------------------------
|
|
154
|
+
# ---- Lines ----------------------------------------------------------------
|
|
155
|
+
# ---------------------------------------------------------------------------
|
|
156
|
+
|
|
157
|
+
def verbatim(line: str) -> bool:
|
|
158
|
+
"""Is this line drawn exactly as it was written (batch 58)?
|
|
159
|
+
|
|
160
|
+
Four columns of indent, or any tab in the indent, and NOTHING is read
|
|
161
|
+
into the line: not a list marker, not a table row, not an inline mark.
|
|
162
|
+
A reported defect rather than a tidy-up -- `x = a ** b ** c` indented
|
|
163
|
+
inside an answer rendered with ` b ` in bold, because batch 53 parsed
|
|
164
|
+
marks on every line regardless of where it sat.
|
|
165
|
+
|
|
166
|
+
Stateless on purpose. GFM decides this with the surrounding list
|
|
167
|
+
context; tracking that would put state across the commit boundary,
|
|
168
|
+
which is where this widget has broken before. The price is named
|
|
169
|
+
rather than hidden: an item nested with FOUR spaces renders flat, and
|
|
170
|
+
one nested with two renders as a nested item.
|
|
171
|
+
|
|
172
|
+
A heading was never exposed -- `_HEADING` is anchored at column 0 --
|
|
173
|
+
and a FENCE is deliberately still exempt. See TECHNICAL_DEBT.md.
|
|
174
|
+
"""
|
|
175
|
+
indent = line[:len(line) - len(line.lstrip())]
|
|
176
|
+
return "\t" in indent or len(indent) >= CODE_INDENT
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def heading(line: str):
|
|
180
|
+
"""The end of this line's ATX marker, or None if it has none.
|
|
181
|
+
|
|
182
|
+
One reading for the renderer and the cap, so a heading held by one
|
|
183
|
+
cannot be drawn as prose by the other. The HOLD_LIMIT guard is half
|
|
184
|
+
of that agreement: a `#` line longer than the limit is not a heading
|
|
185
|
+
on either path, so the released chunk and the replay draw the same
|
|
186
|
+
hashes.
|
|
187
|
+
"""
|
|
188
|
+
if len(line) > HOLD_LIMIT or verbatim(line):
|
|
189
|
+
return None
|
|
190
|
+
match = _HEADING.match(line)
|
|
191
|
+
return match.end() if match else None
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def list_item(line: str):
|
|
195
|
+
"""`(prefix, body, indent)` for a list item, or None.
|
|
196
|
+
|
|
197
|
+
`prefix` is everything before the text -- indent, marker and the gap
|
|
198
|
+
-- and `indent` is the column the text starts at, which is where the
|
|
199
|
+
item's own wrapped rows are padded to. One number for both, because a
|
|
200
|
+
hanging indent that did not line up with the marker's own text would
|
|
201
|
+
be furniture rather than structure.
|
|
202
|
+
|
|
203
|
+
Refused past HOLD_LIMIT for `heading`'s reason, and it is the half
|
|
204
|
+
that makes the stall bounded: a marker with no newline behind it is
|
|
205
|
+
held only until the limit, and past it the line is prose on BOTH
|
|
206
|
+
paths rather than a list item on one of them.
|
|
207
|
+
"""
|
|
208
|
+
if verbatim(line) or len(line) > HOLD_LIMIT:
|
|
209
|
+
return None
|
|
210
|
+
match = _LIST.match(line)
|
|
211
|
+
if match is None:
|
|
212
|
+
return None
|
|
213
|
+
prefix = line[:match.end(3)]
|
|
214
|
+
return prefix, match.group(4), len(prefix)
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
# ---------------------------------------------------------------------------
|
|
218
|
+
# ---- Blocks ---------------------------------------------------------------
|
|
219
|
+
# ---------------------------------------------------------------------------
|
|
220
|
+
|
|
221
|
+
class CodeBlock(tuple):
|
|
222
|
+
"""A fenced block: `(language, code)`, language None when unlabelled.
|
|
223
|
+
|
|
224
|
+
A tuple subclass rather than a NamedTuple or a dataclass, and that is
|
|
225
|
+
load-bearing rather than lazy. `Transcript._render_blocks` asks
|
|
226
|
+
`isinstance(block, tuple)` to mean "a renderable that occupies its own
|
|
227
|
+
rows", which is what drives the newline trimming around it -- so both
|
|
228
|
+
block kinds answering that question keeps ONE rule for the two of them
|
|
229
|
+
instead of a second branch that has to be kept in step. It also unpacks
|
|
230
|
+
exactly as §26's plain `(language, code)` tuple did.
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
__slots__ = ()
|
|
234
|
+
|
|
235
|
+
def __new__(cls, language, code):
|
|
236
|
+
return super().__new__(cls, (language, code))
|
|
237
|
+
|
|
238
|
+
@property
|
|
239
|
+
def language(self):
|
|
240
|
+
return self[0]
|
|
241
|
+
|
|
242
|
+
@property
|
|
243
|
+
def code(self):
|
|
244
|
+
return self[1]
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
class TableBlock(tuple):
|
|
248
|
+
"""A GFM table: `(headers, aligns, rows)`.
|
|
249
|
+
|
|
250
|
+
`aligns` is one of "left" / "center" / "right" per column, in Rich's own
|
|
251
|
+
vocabulary so the renderer hands it straight to `Table.add_column` --
|
|
252
|
+
translating at the boundary rather than storing a markdown spelling the
|
|
253
|
+
widget would have to re-map.
|
|
254
|
+
|
|
255
|
+
Rows are RAGGED-SAFE: short ones are padded and long ones truncated to
|
|
256
|
+
the header's width, which is what GFM specifies and, more to the point,
|
|
257
|
+
what stops one malformed row from raising inside a render.
|
|
258
|
+
"""
|
|
259
|
+
|
|
260
|
+
__slots__ = ()
|
|
261
|
+
|
|
262
|
+
def __new__(cls, headers, aligns, rows):
|
|
263
|
+
return super().__new__(cls, (headers, aligns, rows))
|
|
264
|
+
|
|
265
|
+
@property
|
|
266
|
+
def headers(self):
|
|
267
|
+
return self[0]
|
|
268
|
+
|
|
269
|
+
@property
|
|
270
|
+
def aligns(self):
|
|
271
|
+
return self[1]
|
|
272
|
+
|
|
273
|
+
@property
|
|
274
|
+
def rows(self):
|
|
275
|
+
return self[2]
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _is_fence_line(line: str) -> bool:
|
|
279
|
+
"""Does this line open or close a fenced block?
|
|
280
|
+
|
|
281
|
+
A fence starts the LINE (after whitespace). A ``` mentioned mid-line
|
|
282
|
+
(`Use ```python to start`) is prose mentioning a fence, not a block.
|
|
283
|
+
Any indent counts -- fences are deliberately exempt from `verbatim()`,
|
|
284
|
+
so an indented fence still opens (confirmed, not GFM-strict).
|
|
285
|
+
"""
|
|
286
|
+
return line.lstrip(" \t").startswith(FENCE)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _fence_line_count(text: str) -> int:
|
|
290
|
+
"""How many fence-lines `text` holds. `commit_span`'s parity check,
|
|
291
|
+
kept beside the splitter so the two cannot drift into substring
|
|
292
|
+
counting on one path and line counting on the other."""
|
|
293
|
+
return sum(1 for raw in text.splitlines()
|
|
294
|
+
if _is_fence_line(raw))
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def split_blocks(text: str):
|
|
298
|
+
"""Split markdown into plain strings, `CodeBlock`s and `TableBlock`s.
|
|
299
|
+
|
|
300
|
+
§26's `_split_fences` with a second pass inside its plain parts, and the
|
|
301
|
+
fence half keeps its contract -- an unterminated fence still runs to
|
|
302
|
+
the end of the text (the common case mid-stream), and the plain parts
|
|
303
|
+
still carry their own newlines so the caller's trimming around a
|
|
304
|
+
renderable block keeps working on the reconstructed text.
|
|
305
|
+
|
|
306
|
+
Tables are looked for only OUTSIDE fences, which is why this is nested
|
|
307
|
+
rather than a scan over lines: a pipe table inside a ``` block is source
|
|
308
|
+
code, and highlighting it as one is the whole point of the fence.
|
|
309
|
+
|
|
310
|
+
Fences are LINE constructs (`_is_fence_line`): a ``` mid-line is prose.
|
|
311
|
+
"""
|
|
312
|
+
out, plain, language, code = [], [], None, None
|
|
313
|
+
in_code = False
|
|
314
|
+
for raw in text.splitlines(keepends=True):
|
|
315
|
+
line = _strip_eol(raw)
|
|
316
|
+
if _is_fence_line(line):
|
|
317
|
+
if not in_code:
|
|
318
|
+
if plain:
|
|
319
|
+
out.extend(_split_tables("".join(plain)))
|
|
320
|
+
plain = []
|
|
321
|
+
language = line.lstrip(" \t")[len(FENCE):].strip() or None
|
|
322
|
+
code = []
|
|
323
|
+
in_code = True
|
|
324
|
+
else:
|
|
325
|
+
out.append(CodeBlock(language, "".join(code)))
|
|
326
|
+
language, code, in_code = None, None, False
|
|
327
|
+
if raw.endswith("\n"):
|
|
328
|
+
# The old `text.split(FENCE)` left the closing line's
|
|
329
|
+
# newline in the FOLLOWING plain run, and the
|
|
330
|
+
# renderer's blank-line trimming depends on it.
|
|
331
|
+
plain.append("\n")
|
|
332
|
+
continue
|
|
333
|
+
(code if in_code else plain).append(raw)
|
|
334
|
+
if in_code:
|
|
335
|
+
out.append(CodeBlock(language, "".join(code)))
|
|
336
|
+
elif plain:
|
|
337
|
+
out.extend(_split_tables("".join(plain)))
|
|
338
|
+
return out
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _split_tables(text: str):
|
|
342
|
+
"""One fence-free stretch as plain strings and `TableBlock`s.
|
|
343
|
+
|
|
344
|
+
The plain runs are re-joined with their original line endings, so
|
|
345
|
+
`"".join(the plain parts) + the tables' own source` is the input back --
|
|
346
|
+
which is what lets the caller's blank-line trimming treat a table
|
|
347
|
+
exactly as it treats a fence.
|
|
348
|
+
|
|
349
|
+
THE LAST ROW'S NEWLINE IS LEFT BEHIND, and that is what makes "exactly"
|
|
350
|
+
true rather than approximately. `text.split(FENCE)` puts the newline
|
|
351
|
+
after a closing ``` into the FOLLOWING plain run, so the trimming finds
|
|
352
|
+
two there and keeps one -- a fence draws a blank line after it. A table
|
|
353
|
+
that swallowed its own terminator left only one newline behind, the
|
|
354
|
+
trimming took it, and the paragraph after a table butted straight up
|
|
355
|
+
against the bottom border while the same paragraph after a fence did
|
|
356
|
+
not. Handing the newline back puts both constructs on one rule.
|
|
357
|
+
"""
|
|
358
|
+
lines = text.splitlines(keepends=True)
|
|
359
|
+
out, plain, index = [], [], 0
|
|
360
|
+
while index < len(lines):
|
|
361
|
+
table, consumed = _read_table(lines, index)
|
|
362
|
+
if table is None:
|
|
363
|
+
plain.append(lines[index])
|
|
364
|
+
index += 1
|
|
365
|
+
continue
|
|
366
|
+
if plain:
|
|
367
|
+
out.append("".join(plain))
|
|
368
|
+
plain = []
|
|
369
|
+
out.append(table)
|
|
370
|
+
if lines[index + consumed - 1].endswith("\n"):
|
|
371
|
+
plain.append("\n")
|
|
372
|
+
index += consumed
|
|
373
|
+
if plain:
|
|
374
|
+
out.append("".join(plain))
|
|
375
|
+
return out
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _read_table(lines, start):
|
|
379
|
+
"""`(TableBlock, lines consumed)` if a table starts at `start`, else
|
|
380
|
+
`(None, 0)`.
|
|
381
|
+
|
|
382
|
+
GFM-STRICT, which is the decision rather than an implementation detail:
|
|
383
|
+
a header row is only a header when a delimiter row follows it AND the two
|
|
384
|
+
have the same number of cells. Without that, a shell pipeline, a `|` in
|
|
385
|
+
prose or a column of BNF alternatives all become tables -- and, worse
|
|
386
|
+
than rendering oddly, each one would stall the stream while the cap below
|
|
387
|
+
waited for a table that was never coming.
|
|
388
|
+
"""
|
|
389
|
+
header = _strip_eol(lines[start])
|
|
390
|
+
if not _is_row(header):
|
|
391
|
+
return None, 0
|
|
392
|
+
if start + 1 >= len(lines):
|
|
393
|
+
return None, 0
|
|
394
|
+
delimiter = _strip_eol(lines[start + 1])
|
|
395
|
+
headers = _cells(header)
|
|
396
|
+
aligns = _delimiter_aligns(delimiter, len(headers))
|
|
397
|
+
if aligns is None:
|
|
398
|
+
return None, 0
|
|
399
|
+
|
|
400
|
+
rows, index = [], start + 2
|
|
401
|
+
while index < len(lines):
|
|
402
|
+
body = _strip_eol(lines[index])
|
|
403
|
+
if not _is_row(body):
|
|
404
|
+
break
|
|
405
|
+
rows.append(_fit(_cells(body), len(headers)))
|
|
406
|
+
index += 1
|
|
407
|
+
return TableBlock(headers, aligns, rows), index - start
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def _strip_eol(line: str) -> str:
|
|
411
|
+
return line[:-1] if line.endswith("\n") else line
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _is_row(text: str) -> bool:
|
|
415
|
+
"""A line that could be part of a table: non-blank, and carrying a pipe.
|
|
416
|
+
|
|
417
|
+
A blank line ends a table, which is GFM's rule and also the only thing
|
|
418
|
+
that keeps a table from swallowing the paragraph after it.
|
|
419
|
+
|
|
420
|
+
An INDENTED line is not a row (batch 58), which is what keeps a shell
|
|
421
|
+
pipeline inside a four-space code sample from being drawn as a grid.
|
|
422
|
+
Here rather than at the two callers, so `_read_table` and the cap
|
|
423
|
+
cannot disagree about where a table stops.
|
|
424
|
+
"""
|
|
425
|
+
return bool(text.strip()) and "|" in text and not verbatim(text)
|
|
426
|
+
|
|
427
|
+
|
|
428
|
+
def _trailing_backslashes(text: str) -> int:
|
|
429
|
+
"""Count of trailing backslashes in `text`. The parity decides whether
|
|
430
|
+
a final pipe is escaped (odd) or a delimiter (even)."""
|
|
431
|
+
count = 0
|
|
432
|
+
for char in reversed(text):
|
|
433
|
+
if char != "\\":
|
|
434
|
+
break
|
|
435
|
+
count += 1
|
|
436
|
+
return count
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _cells(text: str):
|
|
440
|
+
"""One row's cells. Outer pipes are optional (GFM allows a table with
|
|
441
|
+
none at all), and `\\|` inside a cell is a literal pipe rather than a
|
|
442
|
+
boundary -- unescaped here so the cell carries what the model meant.
|
|
443
|
+
|
|
444
|
+
Backslash parity, left to right: a double backslash is a literal
|
|
445
|
+
backslash (checked first), so double-backslash-pipe is that backslash
|
|
446
|
+
followed by a DELIMITER, while triple-backslash-pipe is a backslash
|
|
447
|
+
followed by an escaped pipe and stays one cell.
|
|
448
|
+
"""
|
|
449
|
+
body = text.strip()
|
|
450
|
+
if body.startswith("|"):
|
|
451
|
+
body = body[1:]
|
|
452
|
+
if body.endswith("|") and _trailing_backslashes(body[:-1]) % 2 == 0:
|
|
453
|
+
body = body[:-1]
|
|
454
|
+
cells, current, index = [], [], 0
|
|
455
|
+
while index < len(body):
|
|
456
|
+
char = body[index]
|
|
457
|
+
nxt = body[index + 1] if index + 1 < len(body) else ""
|
|
458
|
+
if char == "\\" and nxt == "\\":
|
|
459
|
+
current.append("\\")
|
|
460
|
+
index += 2
|
|
461
|
+
continue
|
|
462
|
+
if char == "\\" and nxt == "|":
|
|
463
|
+
current.append("|")
|
|
464
|
+
index += 2
|
|
465
|
+
continue
|
|
466
|
+
if char == "|":
|
|
467
|
+
cells.append("".join(current).strip())
|
|
468
|
+
current = []
|
|
469
|
+
index += 1
|
|
470
|
+
continue
|
|
471
|
+
current.append(char)
|
|
472
|
+
index += 1
|
|
473
|
+
cells.append("".join(current).strip())
|
|
474
|
+
return cells
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _delimiter_aligns(text: str, columns: int):
|
|
478
|
+
"""The alignments a delimiter row declares, or None if it is not one.
|
|
479
|
+
|
|
480
|
+
The column-count match is part of "is this a delimiter row" rather than
|
|
481
|
+
a separate validation, because that is the question the caller is
|
|
482
|
+
asking: a `|---|` under a three-column header is not a table with a
|
|
483
|
+
mistake in it, it is not a table.
|
|
484
|
+
"""
|
|
485
|
+
if not text.strip():
|
|
486
|
+
return None
|
|
487
|
+
cells = _cells(text)
|
|
488
|
+
if len(cells) != columns:
|
|
489
|
+
return None
|
|
490
|
+
aligns = []
|
|
491
|
+
for cell in cells:
|
|
492
|
+
if not _DELIMITER_CELL.match(cell):
|
|
493
|
+
return None
|
|
494
|
+
left, right = cell.startswith(":"), cell.endswith(":")
|
|
495
|
+
if left and right:
|
|
496
|
+
aligns.append("center")
|
|
497
|
+
elif right:
|
|
498
|
+
aligns.append("right")
|
|
499
|
+
else:
|
|
500
|
+
aligns.append("left")
|
|
501
|
+
return aligns
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _fit(cells, columns):
|
|
505
|
+
if len(cells) < columns:
|
|
506
|
+
return cells + [""] * (columns - len(cells))
|
|
507
|
+
return cells[:columns]
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def _delimiter_possible(prefix: str) -> bool:
|
|
511
|
+
"""Could this PARTIAL line still turn into a delimiter row?
|
|
512
|
+
|
|
513
|
+
The cheap half of the decision the cap needs. A delimiter row is built
|
|
514
|
+
from four characters; one outside that set proves the line above it was
|
|
515
|
+
prose, and the stream can be released without waiting for the newline.
|
|
516
|
+
Deciding this from a prefix is what keeps a `|` inside a sentence from
|
|
517
|
+
holding the whole paragraph that follows it.
|
|
518
|
+
"""
|
|
519
|
+
return all(char in _DELIMITER_CHARS for char in prefix)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
# ---------------------------------------------------------------------------
|
|
523
|
+
# ---- Inline marks ---------------------------------------------------------
|
|
524
|
+
# ---------------------------------------------------------------------------
|
|
525
|
+
|
|
526
|
+
def inline_spans(line: str, *, block: bool = True):
|
|
527
|
+
"""One line as `(text, role)` pairs; role is "" for unmarked text.
|
|
528
|
+
|
|
529
|
+
Deliberately smaller than GFM, and each omission is a decision:
|
|
530
|
+
|
|
531
|
+
- `__strong__` is NOT recognised, and batch 58 did not reverse it.
|
|
532
|
+
Measured against a real CommonMark parser, `call __init__ on it`
|
|
533
|
+
renders a bold `init` under the full rules, anywhere in the line,
|
|
534
|
+
and no flanking rule saves it. `_x_` goes with it: an underscore
|
|
535
|
+
that cannot be trusted at two is not trusted at one either.
|
|
536
|
+
- `*emphasis*` IS recognised as of batch 58, but only where the
|
|
537
|
+
asterisk is not wedged between two word characters. That is what
|
|
538
|
+
keeps `2*3*4` arithmetic and `x*y*z` a filename -- both of which
|
|
539
|
+
full GFM italicises -- while `the *point*` renders.
|
|
540
|
+
- a code span is delimited by single backticks and cannot span a line,
|
|
541
|
+
which is what every model this harness talks to actually emits.
|
|
542
|
+
- a URL is its own span and carries no label. `[docs](https://x)`
|
|
543
|
+
draws every character it was written with; only the URL is marked.
|
|
544
|
+
|
|
545
|
+
`block=False` says this text is a FRAGMENT of a line rather than a
|
|
546
|
+
line -- the tail of a streamed paragraph, or a table cell. Nothing
|
|
547
|
+
that depends on where a line STARTS may fire there: not a heading,
|
|
548
|
+
not the indent rule. It is the fix for a shipped defect where a wrap
|
|
549
|
+
boundary falling just before a `# ` token made the committed fragment
|
|
550
|
+
render as a heading and swallow the hash, which a `/theme` replay then
|
|
551
|
+
put back.
|
|
552
|
+
|
|
553
|
+
A heading contributes its role to the line's UNMARKED spans and leaves
|
|
554
|
+
marked ones alone, so `# The **why**` keeps the strong mark visible
|
|
555
|
+
inside the heading rather than compositing two styles into a third.
|
|
556
|
+
"""
|
|
557
|
+
base = ""
|
|
558
|
+
body = line
|
|
559
|
+
end = heading(line) if block else None
|
|
560
|
+
if end is not None:
|
|
561
|
+
body, base = line[end:], HEADING
|
|
562
|
+
return [(text, role or base)
|
|
563
|
+
for text, role, _s, _e in _spans(body, block=block)]
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def link_spans(text: str, *, targets=()):
|
|
567
|
+
"""`(text, role, target)` triples where the ONLY mark is a URL.
|
|
568
|
+
|
|
569
|
+
The scanner a HARNESS-DRAWN line gets (batch 65), where
|
|
570
|
+
`inline_spans` is the one model prose gets. A tool line is a
|
|
571
|
+
digest -- a shell command, a JSON argument -- not prose, and the
|
|
572
|
+
full grammar does not merely over-decorate it, it EATS
|
|
573
|
+
CHARACTERS. Measured on real calls: `git log --format=%h # `date``
|
|
574
|
+
loses its backticks to a code span, and an MCP filter
|
|
575
|
+
`name~=*test* ~~old~~` renders as `name~=test old`. A digest that
|
|
576
|
+
no longer shows what was run is worse than an inert URL, so
|
|
577
|
+
everything that is not a URL passes through literally.
|
|
578
|
+
|
|
579
|
+
`target` is what a click OPENS and is normally the span's own text,
|
|
580
|
+
which is batch 58's rule unchanged: the URL is its own label, so
|
|
581
|
+
there is nowhere for a destination to differ from what is read.
|
|
582
|
+
|
|
583
|
+
`targets` is the exception, and it exists because a tool line is
|
|
584
|
+
TRUNCATED before it is drawn -- `param_digest` caps a value at 60
|
|
585
|
+
characters, which is shorter than most real documentation URLs.
|
|
586
|
+
An elided run resolves against these candidates, and three things
|
|
587
|
+
have to hold at once:
|
|
588
|
+
|
|
589
|
+
- the run is elided at all, i.e. it ends in characters that are
|
|
590
|
+
not ASCII. Asked that way rather than by naming the ellipsis,
|
|
591
|
+
so this module needs to know nothing about the producer that
|
|
592
|
+
put one there;
|
|
593
|
+
- exactly ONE candidate has the visible part as its prefix. Two
|
|
594
|
+
would make the click a guess;
|
|
595
|
+
- THE WHOLE ORIGIN IS VISIBLE. The invariant that replaces
|
|
596
|
+
"the visible text is the target" is `the visible text tells
|
|
597
|
+
you the origin, and the origin is where it goes`, so a run cut
|
|
598
|
+
off inside its host resolves to nothing and stays literal.
|
|
599
|
+
Truncation may elide a path and a query; it may never elide
|
|
600
|
+
the answer to "where does this take me".
|
|
601
|
+
|
|
602
|
+
A candidate is itself run through `_url`, so `clickable`'s rules --
|
|
603
|
+
ASCII, http(s), and no userinfo -- gate a target exactly as they
|
|
604
|
+
gate a span, and a caller cannot hand in something the reader
|
|
605
|
+
would not have been allowed to click.
|
|
606
|
+
"""
|
|
607
|
+
candidates = [url for value in targets for url in _urls_in(value)]
|
|
608
|
+
out = []
|
|
609
|
+
cursor = at = 0
|
|
610
|
+
while True:
|
|
611
|
+
at = text.find("http", at)
|
|
612
|
+
if at == -1:
|
|
613
|
+
break
|
|
614
|
+
run = _URL.match(text, at)
|
|
615
|
+
if run is None or (at and text[at - 1].isalnum()):
|
|
616
|
+
at += 1
|
|
617
|
+
continue
|
|
618
|
+
url = _trim_url(run.group(0))
|
|
619
|
+
target = url if clickable(url) else _elided(url, candidates)
|
|
620
|
+
if not target:
|
|
621
|
+
at += 1
|
|
622
|
+
continue
|
|
623
|
+
if at > cursor:
|
|
624
|
+
out.append((text[cursor:at], "", ""))
|
|
625
|
+
out.append((url, LINK, target))
|
|
626
|
+
cursor = at = at + len(url)
|
|
627
|
+
if cursor < len(text):
|
|
628
|
+
out.append((text[cursor:], "", ""))
|
|
629
|
+
return out
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
def _urls_in(value: str):
|
|
633
|
+
"""Every armable URL inside one candidate value.
|
|
634
|
+
|
|
635
|
+
Callers hand in whole PARAMETER VALUES rather than URLs, so the URL
|
|
636
|
+
grammar has one definition and it is this module's. A search query
|
|
637
|
+
that happens to contain a URL therefore offers it, and a value that
|
|
638
|
+
contains none offers nothing.
|
|
639
|
+
"""
|
|
640
|
+
if not isinstance(value, str):
|
|
641
|
+
return []
|
|
642
|
+
found = []
|
|
643
|
+
at = 0
|
|
644
|
+
while True:
|
|
645
|
+
at = value.find("http", at)
|
|
646
|
+
if at == -1:
|
|
647
|
+
return found
|
|
648
|
+
span, _opened = _url(value, at)
|
|
649
|
+
if span is None:
|
|
650
|
+
at += 1
|
|
651
|
+
continue
|
|
652
|
+
found.append(span[0])
|
|
653
|
+
at = span[3]
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _elided(run: str, candidates) -> str:
|
|
657
|
+
"""The full URL a truncated `run` stands for, or "".
|
|
658
|
+
|
|
659
|
+
See `link_spans`. The three conditions are in the order that makes
|
|
660
|
+
the cheapest one fail first, and the last of them is the one that
|
|
661
|
+
carries the security property.
|
|
662
|
+
"""
|
|
663
|
+
end = len(run)
|
|
664
|
+
while end and not run[end - 1].isascii():
|
|
665
|
+
end -= 1
|
|
666
|
+
if end == len(run) or not end:
|
|
667
|
+
return ""
|
|
668
|
+
prefix = run[:end]
|
|
669
|
+
matched = [url for url in candidates if url.startswith(prefix)]
|
|
670
|
+
if len(matched) != 1:
|
|
671
|
+
return ""
|
|
672
|
+
target = matched[0]
|
|
673
|
+
origin = len(target) - len(target.split("://", 1)[-1]) \
|
|
674
|
+
+ len(_authority(target))
|
|
675
|
+
return target if len(prefix) >= origin else ""
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
def _spans(text: str, *, block: bool = True):
|
|
679
|
+
"""`(text, role, source start, source end)` for one line."""
|
|
680
|
+
return _scan(text, block=block)[0]
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def _wordish(char: str) -> bool:
|
|
684
|
+
return bool(char) and (char.isalnum() or char == "_")
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def _em_edge(text: str, index: int, *, opening: bool) -> bool:
|
|
688
|
+
"""May the `*` at `index` be that end of an emphasis span?
|
|
689
|
+
|
|
690
|
+
Two rules, and each answers one of batch 53's two objections.
|
|
691
|
+
|
|
692
|
+
WEDGED between two word characters, it is arithmetic or a filename
|
|
693
|
+
and never italics -- measured against a real CommonMark parser, full
|
|
694
|
+
GFM renders `2*3*4` and `x*y*z` with emphasis and this declines to.
|
|
695
|
+
|
|
696
|
+
And an opener must be followed by a non-space, a closer preceded by
|
|
697
|
+
one, which is GFM's own flanking rule and the half that matters for
|
|
698
|
+
STREAMING: without it `a * b * c` would open a mark that never
|
|
699
|
+
closes, and the cap would hold the rest of the paragraph waiting for
|
|
700
|
+
a closer that is not coming.
|
|
701
|
+
"""
|
|
702
|
+
before = text[index - 1] if index else ""
|
|
703
|
+
after = text[index + 1] if index + 1 < len(text) else ""
|
|
704
|
+
if _wordish(before) and _wordish(after):
|
|
705
|
+
return False
|
|
706
|
+
edge = after if opening else before
|
|
707
|
+
return bool(edge) and not edge.isspace()
|
|
708
|
+
|
|
709
|
+
|
|
710
|
+
def _paired(text: str, index: int, token: str, role: str):
|
|
711
|
+
"""`(span, opened)` for a `**`/`~~`/`` ` `` mark starting at `index`.
|
|
712
|
+
|
|
713
|
+
`span` when it closes on this line; `opened` ONLY when nothing closes
|
|
714
|
+
it at all, because that is the one case the cap can usefully hold --
|
|
715
|
+
an empty mark (`****`) is literal and finished, and so is a closer
|
|
716
|
+
further away than HOLD_LIMIT, which is the rule that keeps the
|
|
717
|
+
released stream and the replay drawing the same asterisks.
|
|
718
|
+
"""
|
|
719
|
+
size = len(token)
|
|
720
|
+
close = text.find(token, index + size)
|
|
721
|
+
if close == -1:
|
|
722
|
+
return None, True
|
|
723
|
+
if close > index + size and close - index <= HOLD_LIMIT:
|
|
724
|
+
return (text[index + size:close], role, index, close + size), False
|
|
725
|
+
return None, False
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def _em(text: str, index: int):
|
|
729
|
+
"""`(span, opened)` for a `*emphasis*` span starting at `index`.
|
|
730
|
+
|
|
731
|
+
A loop rather than a `find`, because a candidate closer can be
|
|
732
|
+
refused: `a *b*c` has an asterisk that cannot close, so the mark is
|
|
733
|
+
still OPEN and the cap must keep holding -- the closer may be the one
|
|
734
|
+
that has not arrived yet. Committing it as literal and bolding it on
|
|
735
|
+
the replay is exactly the divergence the hold exists to prevent.
|
|
736
|
+
"""
|
|
737
|
+
if not _em_edge(text, index, opening=True):
|
|
738
|
+
return None, False
|
|
739
|
+
at = index + 1
|
|
740
|
+
while True:
|
|
741
|
+
close = text.find("*", at)
|
|
742
|
+
if close == -1:
|
|
743
|
+
return None, True
|
|
744
|
+
if close > index + 1 and _em_edge(text, close, opening=False):
|
|
745
|
+
if close - index > HOLD_LIMIT:
|
|
746
|
+
return None, False
|
|
747
|
+
return (text[index + 1:close], EM, index, close + 1), False
|
|
748
|
+
at = close + 1
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def clickable(url) -> bool:
|
|
752
|
+
"""May this URL be armed for a click (batch 58)?
|
|
753
|
+
|
|
754
|
+
ASCII ONLY, and that is the security half rather than a tidiness
|
|
755
|
+
one. The design rule is that the visible text IS the target, so
|
|
756
|
+
nothing can hide a destination behind a friendly label -- and a
|
|
757
|
+
homograph is precisely the case where the visible form lies. A URL
|
|
758
|
+
with non-ASCII in it still renders and is still copied by `/copy`; it
|
|
759
|
+
simply is not armed.
|
|
760
|
+
|
|
761
|
+
http and https only: the click ends in the platform's URL handler,
|
|
762
|
+
and a `file://` or a registered custom scheme is not something a
|
|
763
|
+
model gets to reach through a transcript.
|
|
764
|
+
|
|
765
|
+
NO USERINFO, which batch 65 added after measuring that the rule
|
|
766
|
+
above had a hole exactly its own shape.
|
|
767
|
+
`https://accounts.google.com@phish.example/x` is ASCII and https,
|
|
768
|
+
so it was armed -- and it opens phish.example. The visible text IS
|
|
769
|
+
the target, so the letter of the rule held; what a reader takes for
|
|
770
|
+
the destination is the userinfo, and the host is somewhere else.
|
|
771
|
+
That is the same attack `[label](url)` was refused for, arriving
|
|
772
|
+
through the URL's own syntax rather than through a label.
|
|
773
|
+
|
|
774
|
+
NOR A URL THE HARNESS REWROTE, which is the same argument one step
|
|
775
|
+
further and was found by a test rather than by reasoning. The
|
|
776
|
+
userinfo rule alone catches
|
|
777
|
+
`https://user:[REDACTED]@host/...`, and nothing caught
|
|
778
|
+
`https://example.com/x?api_key=[REDACTED]` -- ASCII, https, no
|
|
779
|
+
userinfo, and a perfectly real address that is simply not the one
|
|
780
|
+
that was fetched. Clicking it would send the literal string
|
|
781
|
+
`[REDACTED]` to a host as an api_key.
|
|
782
|
+
|
|
783
|
+
A redacted URL still RENDERS and is still copied, exactly as a
|
|
784
|
+
homograph is: the rule is about what a click may do, never about
|
|
785
|
+
hiding text from the reader.
|
|
786
|
+
"""
|
|
787
|
+
if not isinstance(url, str) or not url.isascii():
|
|
788
|
+
return False
|
|
789
|
+
if any(ord(char) < 32 or ord(char) == 127 for char in url):
|
|
790
|
+
return False
|
|
791
|
+
if not (url.startswith("http://") or url.startswith("https://")):
|
|
792
|
+
return False
|
|
793
|
+
if REDACTED_MARK in url:
|
|
794
|
+
return False
|
|
795
|
+
return "@" not in _authority(url)
|
|
796
|
+
|
|
797
|
+
|
|
798
|
+
def _authority(url: str) -> str:
|
|
799
|
+
"""Everything between `scheme://` and the path, query or fragment.
|
|
800
|
+
|
|
801
|
+
Written out rather than `urllib.parse`d, for this module's rule: it
|
|
802
|
+
imports `re` and nothing else, and a parser that normalises would
|
|
803
|
+
answer about a URL slightly different from the one on screen --
|
|
804
|
+
which is the exact gap the whole link design exists to close.
|
|
805
|
+
"""
|
|
806
|
+
rest = url.split("://", 1)[-1]
|
|
807
|
+
for stop in "/?#":
|
|
808
|
+
rest = rest.split(stop, 1)[0]
|
|
809
|
+
return rest
|
|
810
|
+
|
|
811
|
+
|
|
812
|
+
def _trim_url(url: str) -> str:
|
|
813
|
+
"""A detected URL without the prose punctuation stuck to its end.
|
|
814
|
+
|
|
815
|
+
Balanced brackets survive -- `https://x/a_(b)` keeps its own -- and
|
|
816
|
+
an unbalanced one does not, which is what makes the URL inside
|
|
817
|
+
`[docs](https://x/a)` come out as the URL rather than with the
|
|
818
|
+
markdown link's closing parenthesis welded on.
|
|
819
|
+
"""
|
|
820
|
+
while url:
|
|
821
|
+
last = url[-1]
|
|
822
|
+
if last in _URL_TRIM:
|
|
823
|
+
url = url[:-1]
|
|
824
|
+
continue
|
|
825
|
+
opener = _URL_PAIRS.get(last)
|
|
826
|
+
if opener is not None and url.count(last) > url.count(opener):
|
|
827
|
+
url = url[:-1]
|
|
828
|
+
continue
|
|
829
|
+
break
|
|
830
|
+
return url
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def _url(text: str, index: int):
|
|
834
|
+
"""`(span, opened)` for a bare URL starting at `index`.
|
|
835
|
+
|
|
836
|
+
Never opens: a URL has no closing delimiter, so there is nothing to
|
|
837
|
+
wait for and nothing to hold. A URL that cannot be armed returns no
|
|
838
|
+
span at all and falls through to the literal path, which is how a
|
|
839
|
+
non-ASCII one stays visible without becoming clickable.
|
|
840
|
+
"""
|
|
841
|
+
if index and text[index - 1].isalnum():
|
|
842
|
+
return None, False
|
|
843
|
+
match = _URL.match(text, index)
|
|
844
|
+
if match is None:
|
|
845
|
+
return None, False
|
|
846
|
+
url = _trim_url(match.group(0))
|
|
847
|
+
if not clickable(url):
|
|
848
|
+
return None, False
|
|
849
|
+
return (url, LINK, index, index + len(url)), False
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
def _scan(text: str, *, block: bool = True):
|
|
853
|
+
"""`(spans, first unclosed mark)` for one line, in ONE walk.
|
|
854
|
+
|
|
855
|
+
The two answers come from the same pass because they are the same
|
|
856
|
+
reading: a mark is a span when it closes on this line and an open mark
|
|
857
|
+
when it does not, and two tokenisers asked to agree about that would
|
|
858
|
+
agree right up until they did not. `inline_spans` wants the first half,
|
|
859
|
+
`safe_commit_limit` the second.
|
|
860
|
+
|
|
861
|
+
The source extents are what `width_split` cuts on: a cut INSIDE a mark
|
|
862
|
+
would leave the delimiters unbalanced on both sides of the break, so
|
|
863
|
+
each half would re-parse as literal asterisks while the replay drew one
|
|
864
|
+
styled span. Same divergence the cap exists to prevent, one layer down.
|
|
865
|
+
|
|
866
|
+
A LONE TRAILING `*` counts as open. It is not a mark in this grammar and
|
|
867
|
+
renders as itself, but mid-stream it is the first half of a `**` that
|
|
868
|
+
has not finished arriving -- and the hard-cut branch of `width_split` is
|
|
869
|
+
able to sever exactly there, on a row with no spaces in it.
|
|
870
|
+
|
|
871
|
+
An INDENTED line is one unmarked span and nothing else (batch 58) --
|
|
872
|
+
`verbatim`'s rule, applied HERE so that every caller inherits it:
|
|
873
|
+
`inline_spans` draws it as written, `display_len` measures the
|
|
874
|
+
characters it will really draw, and `width_split` cuts on them.
|
|
875
|
+
"""
|
|
876
|
+
if block and verbatim(text):
|
|
877
|
+
return ([(text, "", 0, len(text))] if text else []), None
|
|
878
|
+
|
|
879
|
+
out, buffer, start, index = [], [], 0, 0
|
|
880
|
+
open_at = None
|
|
881
|
+
length = len(text)
|
|
882
|
+
|
|
883
|
+
def flush(end):
|
|
884
|
+
if buffer:
|
|
885
|
+
out.append(("".join(buffer), "", start, end))
|
|
886
|
+
buffer.clear()
|
|
887
|
+
|
|
888
|
+
while index < length:
|
|
889
|
+
char = text[index]
|
|
890
|
+
span = None
|
|
891
|
+
opened = False
|
|
892
|
+
if char == "h":
|
|
893
|
+
span, opened = _url(text, index)
|
|
894
|
+
elif char == "`":
|
|
895
|
+
span, opened = _paired(text, index, "`", CODE)
|
|
896
|
+
elif text.startswith("~~", index):
|
|
897
|
+
span, opened = _paired(text, index, "~~", STRIKE)
|
|
898
|
+
elif text.startswith("**", index):
|
|
899
|
+
span, opened = _paired(text, index, "**", STRONG)
|
|
900
|
+
elif char == "*":
|
|
901
|
+
span, opened = _em(text, index)
|
|
902
|
+
if span is not None:
|
|
903
|
+
flush(index)
|
|
904
|
+
out.append(span)
|
|
905
|
+
index = span[3]
|
|
906
|
+
continue
|
|
907
|
+
if opened and open_at is None:
|
|
908
|
+
open_at = index
|
|
909
|
+
if not buffer:
|
|
910
|
+
start = index
|
|
911
|
+
buffer.append(char)
|
|
912
|
+
index += 1
|
|
913
|
+
|
|
914
|
+
# A trailing RUN of `*`, `~` or `` ` `` that survived as literal text is
|
|
915
|
+
# the first half of a mark that has not finished arriving -- `**`, or
|
|
916
|
+
# the ``` of a fence. None is a mark in this grammar and all render as
|
|
917
|
+
# themselves, but the hard-cut branch of `width_split` can sever exactly
|
|
918
|
+
# there, on a row with no spaces in it, and a fence severed at its second
|
|
919
|
+
# backtick opens a block the replay never sees.
|
|
920
|
+
if buffer:
|
|
921
|
+
tail = "".join(buffer)
|
|
922
|
+
run = len(tail) - len(tail.rstrip("`*~"))
|
|
923
|
+
if run and (open_at is None or length - run < open_at):
|
|
924
|
+
open_at = length - run
|
|
925
|
+
flush(length)
|
|
926
|
+
return out, open_at
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def display_len(text: str) -> int:
|
|
930
|
+
"""Cells this line will occupy once its marks are rendered.
|
|
931
|
+
|
|
932
|
+
Multi-line input is measured line by line and summed without the
|
|
933
|
+
newlines, which is the only sensible reading for a caller asking "how
|
|
934
|
+
wide is this" -- and the only caller that matters asks it about a single
|
|
935
|
+
line, because the newline rule has already taken everything above it.
|
|
936
|
+
"""
|
|
937
|
+
total = 0
|
|
938
|
+
for line in text.split("\n"):
|
|
939
|
+
# Through `inline_spans`, not `_spans`: a heading's `#` is dropped
|
|
940
|
+
# before it is drawn, so a measurement that counted it would report
|
|
941
|
+
# a width the terminal never sees.
|
|
942
|
+
total += sum(cell_len(span) for span, _role in inline_spans(line))
|
|
943
|
+
return total
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _cut_points(spans):
|
|
947
|
+
"""`(source index, cells drawn before it)` for every position a row may
|
|
948
|
+
break at, in increasing order.
|
|
949
|
+
|
|
950
|
+
A mark offers its two ends and nothing between them: a cut INSIDE one
|
|
951
|
+
would leave the delimiters unbalanced on both sides of the break.
|
|
952
|
+
Unmarked text offers every position inside it. One reading, shared by
|
|
953
|
+
the streaming cut and the wrap, so the two cannot disagree about where
|
|
954
|
+
a row ends.
|
|
955
|
+
"""
|
|
956
|
+
points, cells = [], 0
|
|
957
|
+
for text, role, source, end in spans:
|
|
958
|
+
if role:
|
|
959
|
+
points.append((source, cells))
|
|
960
|
+
cells += cell_len(text)
|
|
961
|
+
points.append((end, cells))
|
|
962
|
+
continue
|
|
963
|
+
for offset, char in enumerate(text):
|
|
964
|
+
points.append((source + offset, cells))
|
|
965
|
+
cells += cell_len(char)
|
|
966
|
+
points.append((end, cells))
|
|
967
|
+
return points
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def width_split(line: str, width: int, *, block: bool = True):
|
|
971
|
+
"""`(commit now, keep buffered)` for one unterminated line.
|
|
972
|
+
|
|
973
|
+
`Transcript._split_committable`'s width rule, measured in RENDERED cells
|
|
974
|
+
and cut at positions that do not fall inside a mark. The shape is
|
|
975
|
+
deliberately the source rule's:
|
|
976
|
+
|
|
977
|
+
- shorter than a row: nothing commits;
|
|
978
|
+
- otherwise cut after the last space that still fits;
|
|
979
|
+
- no space fits: cut at the width, which is what Rich does with a word
|
|
980
|
+
too long for the line (a URL, the case this branch exists for).
|
|
981
|
+
|
|
982
|
+
The one addition is that every candidate position is a span boundary or
|
|
983
|
+
a point inside an unmarked span. A run with no such position -- a single
|
|
984
|
+
`**...**` longer than a row with no spaces in it -- commits nothing and
|
|
985
|
+
waits for the newline, which is the cap's answer applied to the one
|
|
986
|
+
shape the cap itself cannot see.
|
|
987
|
+
|
|
988
|
+
`block` is `inline_spans`' -- False when the text is the tail of a line
|
|
989
|
+
already partly drawn, where the indent rule must not fire because those
|
|
990
|
+
leading spaces are not an indent.
|
|
991
|
+
"""
|
|
992
|
+
spans = _spans(line, block=block)
|
|
993
|
+
if not spans:
|
|
994
|
+
return "", line
|
|
995
|
+
if sum(cell_len(text) for text, _r, _s, _e in spans) <= width:
|
|
996
|
+
return "", line
|
|
997
|
+
|
|
998
|
+
points = _cut_points(spans)
|
|
999
|
+
space = None
|
|
1000
|
+
for index, before in points:
|
|
1001
|
+
if before > width:
|
|
1002
|
+
break
|
|
1003
|
+
if index < len(line) and line[index] == " ":
|
|
1004
|
+
space = index
|
|
1005
|
+
if space is not None:
|
|
1006
|
+
return line[:space + 1], line[space + 1:]
|
|
1007
|
+
|
|
1008
|
+
hard = 0
|
|
1009
|
+
for index, before in points:
|
|
1010
|
+
if before > width:
|
|
1011
|
+
break
|
|
1012
|
+
hard = index
|
|
1013
|
+
if hard <= 0:
|
|
1014
|
+
return "", line
|
|
1015
|
+
return line[:hard], line[hard:]
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def plain_split(line: str, width: int):
|
|
1019
|
+
"""`(commit now, keep buffered)` for thinking text, measured in cells.
|
|
1020
|
+
|
|
1021
|
+
`Transcript._split_committable`'s `marks=False` branch, which is the
|
|
1022
|
+
thinking path. Reasoning renders as prose, marks and all, so parsing
|
|
1023
|
+
marks here would describe a rendering that does not happen -- but the
|
|
1024
|
+
old branch measured `len()` characters, and a double-width glyph is
|
|
1025
|
+
one character and two cells, so thinking held a row Rich had already
|
|
1026
|
+
wrapped. Same shape as `width_split` (last space that fits, else a
|
|
1027
|
+
hard cut at the width Rich would use), over plain characters rather
|
|
1028
|
+
than mark spans. A character is atomic: the cut never lands inside
|
|
1029
|
+
one, so a glyph wider than the row survives whole for Rich to wrap.
|
|
1030
|
+
|
|
1031
|
+
The rule itself is `_plain_cut`, shared with `plain_wrap`, so the
|
|
1032
|
+
streamed commit and the replayed wrap cannot disagree about a row.
|
|
1033
|
+
"""
|
|
1034
|
+
cut = _plain_cut(line, 0, width)
|
|
1035
|
+
if cut is None:
|
|
1036
|
+
return "", line
|
|
1037
|
+
return line[:cut], line[cut:]
|
|
1038
|
+
|
|
1039
|
+
|
|
1040
|
+
def _plain_cut(line: str, start: int, width: int):
|
|
1041
|
+
"""Where `plain_split`'s rule ends the row that begins at `start`, or
|
|
1042
|
+
None when the rest of the line fits in one row or cannot be cut.
|
|
1043
|
+
|
|
1044
|
+
Last space that still fits, else a hard cut at the width -- and never
|
|
1045
|
+
at `start` itself, which is how a glyph wider than the row survives
|
|
1046
|
+
whole. It scans about one row past `start`, never the whole
|
|
1047
|
+
remainder, which is what lets `plain_wrap` walk a long line once.
|
|
1048
|
+
"""
|
|
1049
|
+
cells, space_at = 0, -1
|
|
1050
|
+
for index in range(start, len(line)):
|
|
1051
|
+
char = line[index]
|
|
1052
|
+
if char == " " and cells <= width:
|
|
1053
|
+
space_at = index
|
|
1054
|
+
cells += cell_len(char)
|
|
1055
|
+
if cells > width:
|
|
1056
|
+
break
|
|
1057
|
+
else:
|
|
1058
|
+
return None
|
|
1059
|
+
if space_at > start:
|
|
1060
|
+
return space_at + 1
|
|
1061
|
+
cells = 0
|
|
1062
|
+
for index in range(start, len(line)):
|
|
1063
|
+
char_cells = cell_len(line[index])
|
|
1064
|
+
if cells + char_cells > width:
|
|
1065
|
+
return index if index > start else None
|
|
1066
|
+
cells += char_cells
|
|
1067
|
+
return None
|
|
1068
|
+
|
|
1069
|
+
|
|
1070
|
+
def plain_wrap(line: str, width: int) -> list[str]:
|
|
1071
|
+
"""The rows `line` occupies when drawn at `width` cells, by
|
|
1072
|
+
`plain_split`'s rule -- the thinking span's wrap.
|
|
1073
|
+
|
|
1074
|
+
`plain_split` applied until it commits nothing, but in one walk over
|
|
1075
|
+
the line: calling it in a loop re-measures the whole remainder for
|
|
1076
|
+
every row, which is quadratic on a long paragraph and paid again on
|
|
1077
|
+
every replay (`wrap_display`'s reason, one grammar over). The last row
|
|
1078
|
+
is what that rule would have kept buffered, so a line that fits comes
|
|
1079
|
+
back as itself and an empty line as one empty row.
|
|
1080
|
+
|
|
1081
|
+
A cut at the very END of the line -- a trailing space that overflows
|
|
1082
|
+
the row by its own cell -- closes the last row rather than opening an
|
|
1083
|
+
empty one. `plain_split` hands that empty remainder back, and Rich
|
|
1084
|
+
folds the space into the row it ends, so drawing the remainder put a
|
|
1085
|
+
bar with nothing beside it under every chunk `thinking_delta` commits
|
|
1086
|
+
that way (measured: three extra rows in a streamed run of wide glyphs).
|
|
1087
|
+
"""
|
|
1088
|
+
if width <= 0:
|
|
1089
|
+
return [line]
|
|
1090
|
+
rows, start = [], 0
|
|
1091
|
+
while True:
|
|
1092
|
+
cut = _plain_cut(line, start, width)
|
|
1093
|
+
if cut is None:
|
|
1094
|
+
break
|
|
1095
|
+
rows.append(line[start:cut])
|
|
1096
|
+
start = cut
|
|
1097
|
+
if start < len(line) or not rows:
|
|
1098
|
+
rows.append(line[start:])
|
|
1099
|
+
return rows
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def wrap_display(line: str, width: int, *, block: bool = False):
|
|
1103
|
+
"""The rows `line` occupies when drawn at `width` cells.
|
|
1104
|
+
|
|
1105
|
+
`width_split`'s rule applied until the text runs out, but in ONE pass
|
|
1106
|
+
over ONE points list rather than by calling it in a loop -- which
|
|
1107
|
+
would rescan from the start of the remainder each time and go
|
|
1108
|
+
quadratic on a long line, once per render and again on every `/theme`.
|
|
1109
|
+
|
|
1110
|
+
`block` defaults to False because the only caller wraps a list item's
|
|
1111
|
+
BODY, which is a fragment of its line: the marker has already been
|
|
1112
|
+
taken off the front, so nothing about where the line started applies
|
|
1113
|
+
to it any more.
|
|
1114
|
+
|
|
1115
|
+
A span wider than the row survives whole -- a URL, mostly -- and Rich
|
|
1116
|
+
soft-wraps it afterwards. That is the same answer `width_split` gives
|
|
1117
|
+
the same shape, and it is identical on both paths because both reach
|
|
1118
|
+
it through here.
|
|
1119
|
+
"""
|
|
1120
|
+
if width <= 0 or not line:
|
|
1121
|
+
return [line]
|
|
1122
|
+
points = _cut_points(_spans(line, block=block))
|
|
1123
|
+
if not points:
|
|
1124
|
+
return [line]
|
|
1125
|
+
|
|
1126
|
+
rows, start, base, space = [], 0, 0, None
|
|
1127
|
+
for index, cells in points:
|
|
1128
|
+
if cells - base > width:
|
|
1129
|
+
cut = space if space is not None else None
|
|
1130
|
+
if cut is None and index > start:
|
|
1131
|
+
cut = (index, cells)
|
|
1132
|
+
if cut is not None:
|
|
1133
|
+
rows.append(line[start:cut[0]])
|
|
1134
|
+
start, base, space = cut[0], cut[1], None
|
|
1135
|
+
if index < len(line) and line[index] == " ":
|
|
1136
|
+
space = (index + 1, cells + 1)
|
|
1137
|
+
rows.append(line[start:])
|
|
1138
|
+
return rows
|
|
1139
|
+
|
|
1140
|
+
|
|
1141
|
+
# ---------------------------------------------------------------------------
|
|
1142
|
+
# ---- The commit cap -------------------------------------------------------
|
|
1143
|
+
# ---------------------------------------------------------------------------
|
|
1144
|
+
|
|
1145
|
+
def _opens_line_construct(line: str) -> bool:
|
|
1146
|
+
"""Does this line start a heading or a list item, LENGTH ASIDE?
|
|
1147
|
+
|
|
1148
|
+
The cap's half of `heading` and `list_item`, sharing their patterns
|
|
1149
|
+
but not their HOLD_LIMIT guard -- because the limit is the thing this
|
|
1150
|
+
function's caller ENFORCES. The renderer asks "is this one", and past
|
|
1151
|
+
the limit the answer is no; the cap asks "should I still be waiting",
|
|
1152
|
+
and it has to hold first in order to give up afterwards.
|
|
1153
|
+
"""
|
|
1154
|
+
return not verbatim(line) and bool(_HEADING.match(line)
|
|
1155
|
+
or _LIST.match(line))
|
|
1156
|
+
|
|
1157
|
+
|
|
1158
|
+
def commit_span(committed: str, pending: str):
|
|
1159
|
+
"""`(limit, forced)` -- how much may be drawn, and whether the cap gave up.
|
|
1160
|
+
|
|
1161
|
+
`safe_commit_limit` below is this function's first value and carries
|
|
1162
|
+
the documentation for the cap itself. `forced` is true when a
|
|
1163
|
+
LINE-SCOPED hold was abandoned because it passed HOLD_LIMIT, and it
|
|
1164
|
+
means one thing to the caller: draw this chunk as though it were the
|
|
1165
|
+
middle of a line, because that is what it is. The line it opened is
|
|
1166
|
+
no longer a heading or a list item on either path -- `heading` and
|
|
1167
|
+
`list_item` refuse it at the same limit -- so the released rows and
|
|
1168
|
+
the replayed ones are the same rows.
|
|
1169
|
+
|
|
1170
|
+
A FENCE and a TABLE are never forced. Both must be drawn whole to be
|
|
1171
|
+
drawn correctly: half a fence inverts the fence parity of everything
|
|
1172
|
+
after it, and half a table is two stacked grids. Their holds end at
|
|
1173
|
+
the closing fence, at the first line that is not a row, or at
|
|
1174
|
+
`flush_stream` when the turn does. That is a narrower guarantee than
|
|
1175
|
+
the line-scoped constructs get, and it is the honest one.
|
|
1176
|
+
"""
|
|
1177
|
+
block_at = 0 if _fence_line_count(committed) % 2 == 1 else None
|
|
1178
|
+
fence_open = block_at is not None
|
|
1179
|
+
line_at = None
|
|
1180
|
+
lines = pending.splitlines(keepends=True)
|
|
1181
|
+
in_table = False
|
|
1182
|
+
offset = 0
|
|
1183
|
+
|
|
1184
|
+
for index, raw in enumerate(lines):
|
|
1185
|
+
start = offset
|
|
1186
|
+
offset += len(raw)
|
|
1187
|
+
complete = raw.endswith("\n")
|
|
1188
|
+
line = _strip_eol(raw)
|
|
1189
|
+
|
|
1190
|
+
if fence_open:
|
|
1191
|
+
if _is_fence_line(line):
|
|
1192
|
+
fence_open = False
|
|
1193
|
+
# ONLY once that line has ended. §38's own pin is this
|
|
1194
|
+
# case: a block whose closing fence has no newline after
|
|
1195
|
+
# it yet puts the last newline INSIDE the block, so the
|
|
1196
|
+
# otherwise-committable prefix would draw an unterminated
|
|
1197
|
+
# opener as plain text. The fence is closed for the scan's
|
|
1198
|
+
# purposes; the CAP stays where the opener put it.
|
|
1199
|
+
if complete:
|
|
1200
|
+
block_at = None
|
|
1201
|
+
continue
|
|
1202
|
+
|
|
1203
|
+
# A table ends at the first line that is not a row, and a fence
|
|
1204
|
+
# opener is one -- so this is settled BEFORE the fence branch. The
|
|
1205
|
+
# other order clears the table's hold by never reaching it, and
|
|
1206
|
+
# then the fence's own hold reports a later offset than the table
|
|
1207
|
+
# it silently swallowed.
|
|
1208
|
+
if in_table:
|
|
1209
|
+
if _is_row(line):
|
|
1210
|
+
continue
|
|
1211
|
+
in_table, block_at = False, None
|
|
1212
|
+
|
|
1213
|
+
if _is_fence_line(line):
|
|
1214
|
+
fence_open, block_at = True, _hold(block_at, start)
|
|
1215
|
+
continue
|
|
1216
|
+
|
|
1217
|
+
if _is_row(line):
|
|
1218
|
+
verdict = _table_verdict(lines, index, line)
|
|
1219
|
+
if verdict is None: # cannot tell yet
|
|
1220
|
+
block_at = _hold(block_at, start)
|
|
1221
|
+
continue
|
|
1222
|
+
if verdict:
|
|
1223
|
+
in_table, block_at = True, _hold(block_at, start)
|
|
1224
|
+
continue
|
|
1225
|
+
# settled as prose: fall through and treat it as a normal line
|
|
1226
|
+
|
|
1227
|
+
if not complete:
|
|
1228
|
+
if verbatim(line):
|
|
1229
|
+
continue
|
|
1230
|
+
if _opens_line_construct(line):
|
|
1231
|
+
line_at = _hold(line_at, start)
|
|
1232
|
+
continue
|
|
1233
|
+
mark = _first_open_mark(line)
|
|
1234
|
+
if mark is not None:
|
|
1235
|
+
line_at = _hold(line_at, start + mark)
|
|
1236
|
+
|
|
1237
|
+
# A block hold always sits at or before a line hold -- the branches
|
|
1238
|
+
# that set one `continue` past the branch that sets the other -- so
|
|
1239
|
+
# it wins outright rather than by comparison.
|
|
1240
|
+
if block_at is not None:
|
|
1241
|
+
return block_at, False
|
|
1242
|
+
if line_at is None:
|
|
1243
|
+
return len(pending), False
|
|
1244
|
+
if len(pending) - line_at > HOLD_LIMIT:
|
|
1245
|
+
return len(pending), True
|
|
1246
|
+
return line_at, False
|
|
1247
|
+
|
|
1248
|
+
|
|
1249
|
+
def safe_commit_limit(committed: str, pending: str) -> int:
|
|
1250
|
+
"""How much of `pending` may be drawn now, as an index into it.
|
|
1251
|
+
|
|
1252
|
+
§38 held a streamed chunk while a ``` fence was open, for a reason that
|
|
1253
|
+
was never only about fences: RichLog appends and cannot rewrite a drawn
|
|
1254
|
+
row, so a construct committed in halves renders as its own source and
|
|
1255
|
+
can never be put right. This is that rule as a CAP over four constructs
|
|
1256
|
+
rather than a single rejection.
|
|
1257
|
+
|
|
1258
|
+
| construct | held from | released by |
|
|
1259
|
+
|----------------------|-------------------|------------------------------|
|
|
1260
|
+
| open ``` fence | the opening fence | the closing fence |
|
|
1261
|
+
| table | the header row | a non-row line, or the flush |
|
|
1262
|
+
| heading | the `#` | that line's newline |
|
|
1263
|
+
| list item | the marker | that line's newline |
|
|
1264
|
+
| unclosed `**` / `` ` ``| the mark | its closer, or the newline |
|
|
1265
|
+
|
|
1266
|
+
A LIST ITEM joined that table in batch 58, for the reason the others
|
|
1267
|
+
are in it: its wrapped rows are padded to the marker's own text
|
|
1268
|
+
column, and a fragment committed without the marker can never be
|
|
1269
|
+
indented afterwards.
|
|
1270
|
+
|
|
1271
|
+
The bottom three are ALSO bounded by HOLD_LIMIT. See `commit_span`,
|
|
1272
|
+
which is this function plus the flag saying the bound was reached.
|
|
1273
|
+
|
|
1274
|
+
A CAP rather than a rejection is strictly better than what it replaces:
|
|
1275
|
+
a paragraph sitting in the same buffer as a fence used to wait for the
|
|
1276
|
+
fence to close, and now streams. It is also backward compatible with
|
|
1277
|
+
§38's two pins, where the fence starts at index 0 and the cap is
|
|
1278
|
+
therefore 0.
|
|
1279
|
+
|
|
1280
|
+
A TABLE HAS NO TERMINATOR, so one is held until the model writes a line
|
|
1281
|
+
that is not a row, or until the turn ends -- the same trade the fence
|
|
1282
|
+
hold already makes, and the reason `flush_stream` renders what it has.
|
|
1283
|
+
|
|
1284
|
+
Only the LAST line can be capped for a heading or an unclosed mark: on a
|
|
1285
|
+
line that has ended, an unmatched `**` is literal text and renders that
|
|
1286
|
+
way on both paths, so holding it would be waiting for something that
|
|
1287
|
+
already happened.
|
|
1288
|
+
"""
|
|
1289
|
+
return commit_span(committed, pending)[0]
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
def _hold(open_at, start):
|
|
1293
|
+
"""The EARLIER of two unresolved offsets.
|
|
1294
|
+
|
|
1295
|
+
A cap is one number, and a construct that opens later cannot license
|
|
1296
|
+
drawing through one that is still open before it. Taking the minimum
|
|
1297
|
+
rather than assigning is what a pipe line under a pipe line found: both
|
|
1298
|
+
are undecidable headers, and the second one was reporting its own
|
|
1299
|
+
offset as the limit -- so the first line committed as prose and the
|
|
1300
|
+
delimiter that arrived next had nowhere to attach.
|
|
1301
|
+
"""
|
|
1302
|
+
return start if open_at is None else min(open_at, start)
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
def _table_verdict(lines, index, header):
|
|
1306
|
+
"""True (a table starts here), False (prose), or None (undecidable yet).
|
|
1307
|
+
|
|
1308
|
+
The undecidable case is the one worth naming: a line carrying a pipe is
|
|
1309
|
+
only a header once the line UNDER it has been seen, so it has to be held
|
|
1310
|
+
until then. `_delimiter_possible` is what keeps that hold to a few
|
|
1311
|
+
tokens instead of a paragraph -- the first letter of an ordinary
|
|
1312
|
+
sentence settles it as prose.
|
|
1313
|
+
"""
|
|
1314
|
+
if index + 1 >= len(lines):
|
|
1315
|
+
return None
|
|
1316
|
+
following = lines[index + 1]
|
|
1317
|
+
if following.endswith("\n"):
|
|
1318
|
+
return _delimiter_aligns(_strip_eol(following),
|
|
1319
|
+
len(_cells(header))) is not None
|
|
1320
|
+
if _delimiter_possible(following):
|
|
1321
|
+
return None
|
|
1322
|
+
return False
|
|
1323
|
+
|
|
1324
|
+
|
|
1325
|
+
def _first_open_mark(line: str):
|
|
1326
|
+
"""Index of the first mark on this line that has not been closed, or
|
|
1327
|
+
None. Read off the one tokeniser rather than counted separately, so
|
|
1328
|
+
there is no second reading to agree with it until it does not."""
|
|
1329
|
+
return _scan(line)[1]
|