inkmark 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/Cargo.lock +61 -60
- data/README.md +20 -0
- data/ext/inkmark/Cargo.toml +2 -2
- data/ext/inkmark/src/chunks_by_heading.rs +91 -31
- data/ext/inkmark/src/chunks_by_size.rs +33 -18
- data/ext/inkmark/src/document.rs +22 -20
- data/ext/inkmark/src/image.rs +57 -30
- data/ext/inkmark/src/lib.rs +1 -0
- data/ext/inkmark/src/link.rs +10 -9
- data/ext/inkmark/src/markdown.rs +436 -0
- data/ext/inkmark/src/scheme_filter.rs +31 -6
- data/ext/inkmark/src/truncate.rs +40 -23
- data/lib/inkmark/version.rb +1 -1
- metadata +4 -3
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
//! Markdown serialization through `pulldown-cmark-to-cmark`.
|
|
2
|
+
//!
|
|
3
|
+
//! Every Markdown-emitting API (`to_markdown`, `chunks_by_heading`,
|
|
4
|
+
//! `chunks_by_size`, `truncate_markdown`) funnels through this module so
|
|
5
|
+
//! the serializer always sees the parser options that produced the
|
|
6
|
+
//! events: pulldown-cmark-to-cmark uses them to pick escapes that reparse
|
|
7
|
+
//! to the same document.
|
|
8
|
+
//!
|
|
9
|
+
//! Serializer failures come back as `Err` instead of panicking (a panic
|
|
10
|
+
//! reaches Ruby as `fatal`, which cannot be rescued). Native entry points
|
|
11
|
+
//! turn them into `Inkmark::Error` via [`to_ruby_error`].
|
|
12
|
+
//!
|
|
13
|
+
//! Two entry points: [`write`] streams parser output straight through
|
|
14
|
+
//! (the unfiltered fast path); [`render`] takes filtered events and falls
|
|
15
|
+
//! back to [`tidy_inline_edges`] when the serializer rejects them.
|
|
16
|
+
|
|
17
|
+
use std::collections::VecDeque;
|
|
18
|
+
|
|
19
|
+
use magnus::{prelude::*, Error, ExceptionClass, RClass, Ruby};
|
|
20
|
+
use pulldown_cmark::{CowStr, Event, Options, Tag, TagEnd};
|
|
21
|
+
|
|
22
|
+
use crate::options::Flags;
|
|
23
|
+
|
|
24
|
+
pub use pulldown_cmark_to_cmark::Error as SerializeError;
|
|
25
|
+
|
|
26
|
+
/// Options to serialize a filtered document with: the parser options it
|
|
27
|
+
/// was parsed with, plus what the filters' output needs. The `heading_ids`
|
|
28
|
+
/// filter gives headings an `id`, which Markdown can only express as a
|
|
29
|
+
/// `{ #id }` heading attribute, so heading attributes are switched on for
|
|
30
|
+
/// the serializer (matching 0.2's output) instead of it failing on
|
|
31
|
+
/// headings it cannot otherwise preserve.
|
|
32
|
+
pub fn serializer_options(cm_opts: Options, flags: &Flags) -> Options {
|
|
33
|
+
if flags.heading_ids {
|
|
34
|
+
cm_opts | Options::ENABLE_HEADING_ATTRIBUTES
|
|
35
|
+
} else {
|
|
36
|
+
cm_opts
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/// Serialize unfiltered parser output into `buf`. `md_opts` must be the
|
|
41
|
+
/// options the events were parsed with.
|
|
42
|
+
pub fn write<'a, I>(events: I, buf: &mut String, md_opts: Options) -> Result<(), SerializeError>
|
|
43
|
+
where
|
|
44
|
+
I: Iterator<Item = Event<'a>>,
|
|
45
|
+
{
|
|
46
|
+
pulldown_cmark_to_cmark::cmark(InertHtmlBlocks::new(events), buf, md_opts)?;
|
|
47
|
+
Ok(())
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/// Serialize filtered events into a new String. `md_opts` comes from
|
|
51
|
+
/// [`serializer_options`].
|
|
52
|
+
///
|
|
53
|
+
/// Filters can leave shapes the serializer cannot write (see
|
|
54
|
+
/// [`tidy_inline_edges`]). Parser output never has them, so events are
|
|
55
|
+
/// reshaped only after a rejection, then serialized once more: output that
|
|
56
|
+
/// serializes as-is is never touched.
|
|
57
|
+
pub fn render(events: &[Event<'_>], md_opts: Options) -> Result<String, SerializeError> {
|
|
58
|
+
let mut buf = String::new();
|
|
59
|
+
if write(events.iter().cloned(), &mut buf, md_opts).is_ok() {
|
|
60
|
+
return Ok(buf);
|
|
61
|
+
}
|
|
62
|
+
let mut events = events.to_vec();
|
|
63
|
+
tidy_inline_edges(&mut events);
|
|
64
|
+
let mut buf = String::new();
|
|
65
|
+
write(events.into_iter(), &mut buf, md_opts)?;
|
|
66
|
+
Ok(buf)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/// Blank line that rejoins top-level blocks serialized one at a time.
|
|
70
|
+
/// Without it, adjacent paragraphs would merge on reparse.
|
|
71
|
+
pub const BLOCK_SEPARATOR: &str = "\n\n";
|
|
72
|
+
|
|
73
|
+
/// Serialize one top-level block for joining with [`BLOCK_SEPARATOR`]:
|
|
74
|
+
/// rendered alone, minus the newlines it starts or ends with.
|
|
75
|
+
pub fn render_block(events: &[Event<'_>], md_opts: Options) -> Result<String, SerializeError> {
|
|
76
|
+
Ok(render(events, md_opts)?.trim_matches('\n').to_owned())
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/// Convert a serializer failure into an `Inkmark::Error` exception.
|
|
80
|
+
///
|
|
81
|
+
/// The class is looked up on each call rather than cached: the extension
|
|
82
|
+
/// keeps no Ruby `VALUE`s in process-global state (see `rb_ext_ractor_safe`
|
|
83
|
+
/// in `lib.rs`), and this path only runs on failure.
|
|
84
|
+
pub fn to_ruby_error(ruby: &Ruby, err: SerializeError) -> Error {
|
|
85
|
+
let class = ruby
|
|
86
|
+
.class_object()
|
|
87
|
+
.const_get::<_, RClass>("Inkmark")
|
|
88
|
+
.and_then(|inkmark| inkmark.const_get::<_, ExceptionClass>("Error"));
|
|
89
|
+
match class {
|
|
90
|
+
Ok(class) => Error::new(class, format!("markdown serialization failed: {err}")),
|
|
91
|
+
Err(lookup_failed) => lookup_failed,
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/// Inline containers written as delimiter runs (`*`, `**`, `~~`, `^`, `~`).
|
|
96
|
+
/// CommonMark only recognizes a run that hugs non-whitespace, so these
|
|
97
|
+
/// cannot be empty or start or end with whitespace or a line break.
|
|
98
|
+
fn is_delimited_start(event: Option<&Event<'_>>) -> bool {
|
|
99
|
+
matches!(
|
|
100
|
+
event,
|
|
101
|
+
Some(Event::Start(
|
|
102
|
+
Tag::Emphasis | Tag::Strong | Tag::Strikethrough | Tag::Superscript | Tag::Subscript
|
|
103
|
+
))
|
|
104
|
+
)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
fn is_delimited_end(end: &TagEnd) -> bool {
|
|
108
|
+
matches!(
|
|
109
|
+
end,
|
|
110
|
+
TagEnd::Emphasis
|
|
111
|
+
| TagEnd::Strong
|
|
112
|
+
| TagEnd::Strikethrough
|
|
113
|
+
| TagEnd::Superscript
|
|
114
|
+
| TagEnd::Subscript
|
|
115
|
+
)
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/// Block containers: a line break cannot open or close one.
|
|
119
|
+
fn is_block_start(event: Option<&Event<'_>>) -> bool {
|
|
120
|
+
matches!(event, Some(Event::Start(tag)) if !matches!(tag,
|
|
121
|
+
Tag::Emphasis | Tag::Strong | Tag::Strikethrough | Tag::Superscript
|
|
122
|
+
| Tag::Subscript | Tag::Link { .. } | Tag::Image { .. }))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
fn is_block_end(end: &TagEnd) -> bool {
|
|
126
|
+
!is_delimited_end(end) && !matches!(end, TagEnd::Link | TagEnd::Image)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
fn is_break(event: Option<&Event<'_>>) -> bool {
|
|
130
|
+
matches!(event, Some(Event::SoftBreak | Event::HardBreak))
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/// What the parser strips at both ends of every line of inline content.
|
|
134
|
+
fn is_line_space(c: char) -> bool {
|
|
135
|
+
c == ' ' || c == '\t'
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/// A line of inline content starts after this event.
|
|
139
|
+
fn opens_line(event: Option<&Event<'_>>) -> bool {
|
|
140
|
+
is_break(event)
|
|
141
|
+
|| matches!(
|
|
142
|
+
event,
|
|
143
|
+
Some(Event::Start(
|
|
144
|
+
Tag::Paragraph | Tag::Heading { .. } | Tag::TableCell | Tag::Item
|
|
145
|
+
))
|
|
146
|
+
)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/// A line of inline content ends at this tag.
|
|
150
|
+
fn closes_line(end: &TagEnd) -> bool {
|
|
151
|
+
matches!(
|
|
152
|
+
end,
|
|
153
|
+
TagEnd::Paragraph | TagEnd::Heading(_) | TagEnd::TableCell | TagEnd::Item
|
|
154
|
+
)
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
fn last_text_ends_with_line_space(out: &[Event<'_>]) -> bool {
|
|
158
|
+
matches!(out.last(), Some(Event::Text(t)) if t.ends_with(is_line_space))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
fn trim_last_text_end(out: &mut Vec<Event<'_>>) {
|
|
162
|
+
if let Some(Event::Text(text)) = out.pop() {
|
|
163
|
+
let body = text.trim_end_matches(is_line_space);
|
|
164
|
+
if !body.is_empty() {
|
|
165
|
+
out.push(Event::Text(CowStr::from(body.to_owned())));
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/// Reshape filtered events the serializer could not otherwise write.
|
|
171
|
+
///
|
|
172
|
+
/// Filters can leave a delimited container with whitespace at an edge or
|
|
173
|
+
/// with nothing inside, typically by unwrapping a disallowed link or image
|
|
174
|
+
/// so its text lands directly inside `**...**`. Line breaks and whitespace
|
|
175
|
+
/// at an edge move outside the container, emptied containers are dropped,
|
|
176
|
+
/// a line break left opening or closing a block is dropped, and spaces or
|
|
177
|
+
/// tabs left at the start or end of a line are trimmed. Rendering is
|
|
178
|
+
/// unchanged (`<strong>a </strong>` displays like `<strong>a</strong> `).
|
|
179
|
+
fn tidy_inline_edges(events: &mut Vec<Event<'_>>) {
|
|
180
|
+
// Each pass fixes one nesting level per edge; repeat until stable.
|
|
181
|
+
// Every change shortens the list or moves an event strictly outward,
|
|
182
|
+
// so this terminates.
|
|
183
|
+
while tidy_pass(events) {}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
fn tidy_pass(events: &mut Vec<Event<'_>>) -> bool {
|
|
187
|
+
let mut changed = false;
|
|
188
|
+
let mut out: Vec<Event<'_>> = Vec::with_capacity(events.len());
|
|
189
|
+
for event in events.drain(..) {
|
|
190
|
+
match event {
|
|
191
|
+
Event::End(end) if is_delimited_end(&end) && is_delimited_start(out.last()) => {
|
|
192
|
+
out.pop();
|
|
193
|
+
changed = true;
|
|
194
|
+
}
|
|
195
|
+
Event::End(end) if is_delimited_end(&end) && is_break(out.last()) => {
|
|
196
|
+
let brk = out.pop().expect("checked non-empty");
|
|
197
|
+
out.push(Event::End(end));
|
|
198
|
+
out.push(brk);
|
|
199
|
+
changed = true;
|
|
200
|
+
}
|
|
201
|
+
Event::End(end)
|
|
202
|
+
if is_delimited_end(&end)
|
|
203
|
+
&& matches!(out.last(), Some(Event::Text(t)) if t.ends_with(char::is_whitespace)) =>
|
|
204
|
+
{
|
|
205
|
+
let Some(Event::Text(text)) = out.pop() else {
|
|
206
|
+
unreachable!("checked above")
|
|
207
|
+
};
|
|
208
|
+
let body = text.trim_end();
|
|
209
|
+
let space = text[body.len()..].to_owned();
|
|
210
|
+
if !body.is_empty() {
|
|
211
|
+
out.push(Event::Text(CowStr::from(body.to_owned())));
|
|
212
|
+
}
|
|
213
|
+
out.push(Event::End(end));
|
|
214
|
+
out.push(Event::Text(CowStr::from(space)));
|
|
215
|
+
changed = true;
|
|
216
|
+
}
|
|
217
|
+
Event::End(end) if is_block_end(&end) && is_break(out.last()) => {
|
|
218
|
+
out.pop();
|
|
219
|
+
out.push(Event::End(end));
|
|
220
|
+
changed = true;
|
|
221
|
+
}
|
|
222
|
+
Event::SoftBreak | Event::HardBreak if is_delimited_start(out.last()) => {
|
|
223
|
+
let start = out.pop().expect("checked non-empty");
|
|
224
|
+
out.push(event);
|
|
225
|
+
out.push(start);
|
|
226
|
+
changed = true;
|
|
227
|
+
}
|
|
228
|
+
Event::SoftBreak | Event::HardBreak if is_block_start(out.last()) => {
|
|
229
|
+
changed = true;
|
|
230
|
+
}
|
|
231
|
+
Event::Text(text)
|
|
232
|
+
if text.starts_with(char::is_whitespace) && is_delimited_start(out.last()) =>
|
|
233
|
+
{
|
|
234
|
+
let start = out.pop().expect("checked non-empty");
|
|
235
|
+
let body = text.trim_start();
|
|
236
|
+
let space = text[..text.len() - body.len()].to_owned();
|
|
237
|
+
out.push(Event::Text(CowStr::from(space)));
|
|
238
|
+
out.push(start);
|
|
239
|
+
if !body.is_empty() {
|
|
240
|
+
out.push(Event::Text(CowStr::from(body.to_owned())));
|
|
241
|
+
}
|
|
242
|
+
changed = true;
|
|
243
|
+
}
|
|
244
|
+
Event::End(end) if closes_line(&end) && last_text_ends_with_line_space(&out) => {
|
|
245
|
+
trim_last_text_end(&mut out);
|
|
246
|
+
out.push(Event::End(end));
|
|
247
|
+
changed = true;
|
|
248
|
+
}
|
|
249
|
+
Event::SoftBreak | Event::HardBreak if last_text_ends_with_line_space(&out) => {
|
|
250
|
+
trim_last_text_end(&mut out);
|
|
251
|
+
out.push(event);
|
|
252
|
+
changed = true;
|
|
253
|
+
}
|
|
254
|
+
Event::Text(text) if text.starts_with(is_line_space) && opens_line(out.last()) => {
|
|
255
|
+
let body = text.trim_start_matches(is_line_space);
|
|
256
|
+
if !body.is_empty() {
|
|
257
|
+
out.push(Event::Text(CowStr::from(body.to_owned())));
|
|
258
|
+
}
|
|
259
|
+
changed = true;
|
|
260
|
+
}
|
|
261
|
+
other => out.push(other),
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
*events = out;
|
|
265
|
+
changed
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/// Keeps suppressed raw HTML inert in Markdown output.
|
|
269
|
+
///
|
|
270
|
+
/// `suppress_raw_html` turns `Html` events into `Text` but leaves the
|
|
271
|
+
/// `HtmlBlock` container around them, which is what the HTML renderer
|
|
272
|
+
/// wants (escaped text, no `<p>`). The Markdown serializer, though, writes
|
|
273
|
+
/// an `HtmlBlock`'s contents verbatim, so that text would reparse as live
|
|
274
|
+
/// HTML. Blocks whose contents start with `Text` are rewritten as a
|
|
275
|
+
/// paragraph, one trimmed line per `SoftBreak`, so the serializer escapes
|
|
276
|
+
/// them. Blocks of real `Html` events (`raw_html: true`) pass through.
|
|
277
|
+
struct InertHtmlBlocks<'a, I: Iterator<Item = Event<'a>>> {
|
|
278
|
+
inner: std::iter::Peekable<I>,
|
|
279
|
+
pending: VecDeque<Event<'a>>,
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
impl<'a, I: Iterator<Item = Event<'a>>> InertHtmlBlocks<'a, I> {
|
|
283
|
+
fn new(inner: I) -> Self {
|
|
284
|
+
Self {
|
|
285
|
+
inner: inner.peekable(),
|
|
286
|
+
pending: VecDeque::new(),
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/// Consume the rest of a suppressed HTML block (through its `End`) and
|
|
291
|
+
/// queue it as a paragraph.
|
|
292
|
+
fn queue_as_paragraph(&mut self) {
|
|
293
|
+
let mut lines: Vec<String> = Vec::new();
|
|
294
|
+
loop {
|
|
295
|
+
match self.inner.next() {
|
|
296
|
+
// An `Html` event in a block that was otherwise suppressed
|
|
297
|
+
// is escaped too: staying inert is the safe default.
|
|
298
|
+
Some(Event::Text(t)) | Some(Event::Html(t)) => lines.extend(
|
|
299
|
+
t.split('\n')
|
|
300
|
+
.map(str::trim)
|
|
301
|
+
.filter(|l| !l.is_empty())
|
|
302
|
+
.map(str::to_owned),
|
|
303
|
+
),
|
|
304
|
+
Some(Event::End(TagEnd::HtmlBlock)) | None => break,
|
|
305
|
+
Some(_) => {}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if lines.is_empty() {
|
|
309
|
+
return;
|
|
310
|
+
}
|
|
311
|
+
self.pending.push_back(Event::Start(Tag::Paragraph));
|
|
312
|
+
for (i, line) in lines.into_iter().enumerate() {
|
|
313
|
+
if i > 0 {
|
|
314
|
+
self.pending.push_back(Event::SoftBreak);
|
|
315
|
+
}
|
|
316
|
+
self.pending.push_back(Event::Text(CowStr::from(line)));
|
|
317
|
+
}
|
|
318
|
+
self.pending.push_back(Event::End(TagEnd::Paragraph));
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
impl<'a, I: Iterator<Item = Event<'a>>> Iterator for InertHtmlBlocks<'a, I> {
|
|
323
|
+
type Item = Event<'a>;
|
|
324
|
+
|
|
325
|
+
fn next(&mut self) -> Option<Event<'a>> {
|
|
326
|
+
loop {
|
|
327
|
+
if let Some(event) = self.pending.pop_front() {
|
|
328
|
+
return Some(event);
|
|
329
|
+
}
|
|
330
|
+
match self.inner.next()? {
|
|
331
|
+
Event::Start(Tag::HtmlBlock)
|
|
332
|
+
if matches!(self.inner.peek(), Some(Event::Text(_))) =>
|
|
333
|
+
{
|
|
334
|
+
// May queue nothing (all-blank block); loop either way.
|
|
335
|
+
self.queue_as_paragraph();
|
|
336
|
+
}
|
|
337
|
+
event => return Some(event),
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
#[cfg(test)]
|
|
344
|
+
mod tests {
|
|
345
|
+
use super::*;
|
|
346
|
+
use pulldown_cmark::Parser;
|
|
347
|
+
|
|
348
|
+
fn suppressed(source: &str) -> Vec<Event<'_>> {
|
|
349
|
+
Parser::new_ext(source, Options::empty())
|
|
350
|
+
.map(|e| match e {
|
|
351
|
+
Event::Html(h) | Event::InlineHtml(h) => Event::Text(h),
|
|
352
|
+
other => other,
|
|
353
|
+
})
|
|
354
|
+
.collect()
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
fn reparse_has_html(markdown: &str) -> bool {
|
|
358
|
+
Parser::new_ext(markdown, Options::empty())
|
|
359
|
+
.any(|e| matches!(e, Event::Html(_) | Event::InlineHtml(_)))
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
#[test]
|
|
363
|
+
fn suppressed_html_block_stays_inert() {
|
|
364
|
+
let md = render(
|
|
365
|
+
&suppressed("<div onclick=\"x()\">\n*inside*\n</div>\n"),
|
|
366
|
+
Options::empty(),
|
|
367
|
+
)
|
|
368
|
+
.unwrap();
|
|
369
|
+
assert!(!reparse_has_html(&md), "reparsed as live HTML: {md:?}");
|
|
370
|
+
assert!(md.contains("div"), "lost the escaped text: {md:?}");
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
#[test]
|
|
374
|
+
fn suppressed_html_block_drops_blank_lines() {
|
|
375
|
+
// HTML block types 1-5 (here <script>) may contain blank lines,
|
|
376
|
+
// which would end a paragraph on reparse.
|
|
377
|
+
let md = render(
|
|
378
|
+
&suppressed("<script>\n\nalert(1)\n</script>\n"),
|
|
379
|
+
Options::empty(),
|
|
380
|
+
)
|
|
381
|
+
.unwrap();
|
|
382
|
+
assert!(!reparse_has_html(&md), "reparsed as live HTML: {md:?}");
|
|
383
|
+
assert!(md.contains("alert(1)"), "lost the escaped text: {md:?}");
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
#[test]
|
|
387
|
+
fn live_html_block_passes_through() {
|
|
388
|
+
let events: Vec<Event> =
|
|
389
|
+
Parser::new_ext("<div>\nraw\n</div>\n", Options::empty()).collect();
|
|
390
|
+
let md = render(&events, Options::empty()).unwrap();
|
|
391
|
+
assert!(md.contains("<div>"), "raw HTML was rewritten: {md:?}");
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/// Parse `source` and drop every link's Start/End, as the allowlist
|
|
395
|
+
/// filters do when they unwrap a disallowed link to its text.
|
|
396
|
+
fn unwrapped_links(source: &str) -> Vec<Event<'_>> {
|
|
397
|
+
Parser::new_ext(source, Options::empty())
|
|
398
|
+
.filter(|e| !matches!(e, Event::Start(Tag::Link { .. }) | Event::End(TagEnd::Link)))
|
|
399
|
+
.collect()
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
#[test]
|
|
403
|
+
fn unwrapped_link_edges_inside_emphasis_serialize() {
|
|
404
|
+
for source in [
|
|
405
|
+
"**[text\n](u)**",
|
|
406
|
+
"**[\ntext](u)**",
|
|
407
|
+
"**[text ](u)**",
|
|
408
|
+
"**[ text](u)**",
|
|
409
|
+
"**_[text\n](u)_**",
|
|
410
|
+
"a **[text ](u)** b",
|
|
411
|
+
] {
|
|
412
|
+
let md = render(&unwrapped_links(source), Options::empty())
|
|
413
|
+
.unwrap_or_else(|e| panic!("{source:?} failed: {e}"));
|
|
414
|
+
let strong = Parser::new_ext(&md, Options::empty())
|
|
415
|
+
.any(|e| matches!(e, Event::Start(Tag::Strong)));
|
|
416
|
+
assert!(strong && md.contains("text"), "{source:?} => {md:?}");
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
#[test]
|
|
421
|
+
fn emphasis_emptied_by_unwrapping_is_dropped() {
|
|
422
|
+
let md = render(&unwrapped_links("a *[](u)* b"), Options::empty()).unwrap();
|
|
423
|
+
assert!(!md.contains('*'), "{md:?}");
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
#[test]
|
|
427
|
+
fn unbalanced_events_return_an_error_instead_of_panicking() {
|
|
428
|
+
let events = vec![
|
|
429
|
+
Event::Start(Tag::Paragraph),
|
|
430
|
+
Event::Text(CowStr::Borrowed("x")),
|
|
431
|
+
Event::End(TagEnd::Paragraph),
|
|
432
|
+
Event::End(TagEnd::BlockQuote(None)),
|
|
433
|
+
];
|
|
434
|
+
assert!(render(&events, Options::empty()).is_err());
|
|
435
|
+
}
|
|
436
|
+
}
|
|
@@ -35,8 +35,10 @@ pub struct SchemeFilter<'e, 's, I: Iterator<Item = Event<'e>>> {
|
|
|
35
35
|
// disallows nested links so this is usually 1, but the counter keeps
|
|
36
36
|
// us correct on malformed / extension-emitted streams.
|
|
37
37
|
skipping_link_depth: usize,
|
|
38
|
-
// State for in-progress image drop
|
|
39
|
-
|
|
38
|
+
// State for in-progress image drop, counted like links: an image can
|
|
39
|
+
// nest in another image's alt text (`](y)`). 0 = not
|
|
40
|
+
// skipping.
|
|
41
|
+
skipping_image_depth: usize,
|
|
40
42
|
image_alt: String,
|
|
41
43
|
}
|
|
42
44
|
|
|
@@ -51,7 +53,7 @@ impl<'e, 's, I: Iterator<Item = Event<'e>>> SchemeFilter<'e, 's, I> {
|
|
|
51
53
|
link_allowed,
|
|
52
54
|
image_allowed,
|
|
53
55
|
skipping_link_depth: 0,
|
|
54
|
-
|
|
56
|
+
skipping_image_depth: 0,
|
|
55
57
|
image_alt: String::new(),
|
|
56
58
|
}
|
|
57
59
|
}
|
|
@@ -63,10 +65,17 @@ impl<'e, 's, I: Iterator<Item = Event<'e>>> Iterator for SchemeFilter<'e, 's, I>
|
|
|
63
65
|
fn next(&mut self) -> Option<Event<'e>> {
|
|
64
66
|
loop {
|
|
65
67
|
// Image skipping mode
|
|
66
|
-
if self.
|
|
68
|
+
if self.skipping_image_depth > 0 {
|
|
67
69
|
match self.inner.next()? {
|
|
70
|
+
Event::Start(Tag::Image { .. }) => {
|
|
71
|
+
self.skipping_image_depth += 1;
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
68
74
|
Event::End(TagEnd::Image) => {
|
|
69
|
-
self.
|
|
75
|
+
self.skipping_image_depth -= 1;
|
|
76
|
+
if self.skipping_image_depth > 0 {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
70
79
|
if !self.image_alt.is_empty() {
|
|
71
80
|
let alt = std::mem::take(&mut self.image_alt);
|
|
72
81
|
return Some(Event::Text(CowStr::Boxed(alt.into_boxed_str())));
|
|
@@ -117,7 +126,7 @@ impl<'e, 's, I: Iterator<Item = Event<'e>>> Iterator for SchemeFilter<'e, 's, I>
|
|
|
117
126
|
.image_allowed
|
|
118
127
|
.is_some_and(|s| !is_scheme_allowed(dest_url, s)) =>
|
|
119
128
|
{
|
|
120
|
-
self.
|
|
129
|
+
self.skipping_image_depth = 1;
|
|
121
130
|
self.image_alt.clear();
|
|
122
131
|
continue;
|
|
123
132
|
}
|
|
@@ -162,6 +171,22 @@ mod tests {
|
|
|
162
171
|
assert!(matches!(out[0], Event::Start(Tag::Link { .. })));
|
|
163
172
|
}
|
|
164
173
|
|
|
174
|
+
#[test]
|
|
175
|
+
fn drops_blocked_image_with_an_image_nested_in_its_alt_text() {
|
|
176
|
+
let source = "](http://x.com/2)";
|
|
177
|
+
let events: Vec<Event> = pulldown_cmark::Parser::new(source).collect();
|
|
178
|
+
let allowed = schemes(&["https"]);
|
|
179
|
+
let out = run(events, None, Some(&allowed));
|
|
180
|
+
assert!(
|
|
181
|
+
!out.iter().any(|e| matches!(
|
|
182
|
+
e,
|
|
183
|
+
Event::Start(Tag::Image { .. }) | Event::End(TagEnd::Image)
|
|
184
|
+
)),
|
|
185
|
+
"stray image event: {out:?}"
|
|
186
|
+
);
|
|
187
|
+
assert!(out.contains(&Event::Text(CowStr::Borrowed("foo bar"))));
|
|
188
|
+
}
|
|
189
|
+
|
|
165
190
|
#[test]
|
|
166
191
|
fn drops_blocked_link_keeps_text() {
|
|
167
192
|
let events = vec![
|
data/ext/inkmark/src/truncate.rs
CHANGED
|
@@ -11,6 +11,7 @@ use pulldown_cmark::Event;
|
|
|
11
11
|
use unicode_segmentation::UnicodeSegmentation;
|
|
12
12
|
|
|
13
13
|
use crate::document::{apply_filters, content_events};
|
|
14
|
+
use crate::markdown::{self, SerializeError};
|
|
14
15
|
use crate::options::{build_options, Flags};
|
|
15
16
|
|
|
16
17
|
/// What kind of boundary to cut at.
|
|
@@ -43,10 +44,14 @@ pub fn truncate_source(
|
|
|
43
44
|
cm_opts: pulldown_cmark::Options,
|
|
44
45
|
flags: &Flags,
|
|
45
46
|
params: &TruncateParams,
|
|
46
|
-
) -> String {
|
|
47
|
+
) -> Result<String, SerializeError> {
|
|
47
48
|
let events: Vec<Event> = content_events(source, cm_opts).collect();
|
|
48
49
|
let events = apply_filters(events, flags);
|
|
49
|
-
truncate_events(
|
|
50
|
+
truncate_events(
|
|
51
|
+
&events,
|
|
52
|
+
params,
|
|
53
|
+
markdown::serializer_options(cm_opts, flags),
|
|
54
|
+
)
|
|
50
55
|
}
|
|
51
56
|
|
|
52
57
|
/// Ruby-facing entry point. Expects `params_hash` to contain
|
|
@@ -61,7 +66,7 @@ pub fn native_truncate_markdown(
|
|
|
61
66
|
) -> Result<String, Error> {
|
|
62
67
|
let (cm_opts, flags) = build_options(ruby, opts_hash)?;
|
|
63
68
|
let params = parse_params(ruby, params_hash)?;
|
|
64
|
-
|
|
69
|
+
truncate_source(&source, cm_opts, &flags, ¶ms).map_err(|e| markdown::to_ruby_error(ruby, e))
|
|
65
70
|
}
|
|
66
71
|
|
|
67
72
|
pub fn parse_params(ruby: &Ruby, hash: RHash) -> Result<TruncateParams, Error> {
|
|
@@ -85,14 +90,22 @@ pub fn parse_params(ruby: &Ruby, hash: RHash) -> Result<TruncateParams, Error> {
|
|
|
85
90
|
}
|
|
86
91
|
|
|
87
92
|
/// Core truncation over a filter-applied event slice.
|
|
88
|
-
pub fn truncate_events(
|
|
93
|
+
pub fn truncate_events(
|
|
94
|
+
events: &[Event<'_>],
|
|
95
|
+
params: &TruncateParams,
|
|
96
|
+
md_opts: pulldown_cmark::Options,
|
|
97
|
+
) -> Result<String, SerializeError> {
|
|
89
98
|
match params.at {
|
|
90
|
-
TruncateAt::Block => truncate_at_block(events, params),
|
|
91
|
-
TruncateAt::Word => truncate_at_word(events, params),
|
|
99
|
+
TruncateAt::Block => truncate_at_block(events, params, md_opts),
|
|
100
|
+
TruncateAt::Word => truncate_at_word(events, params, md_opts),
|
|
92
101
|
}
|
|
93
102
|
}
|
|
94
103
|
|
|
95
|
-
fn truncate_at_block(
|
|
104
|
+
fn truncate_at_block(
|
|
105
|
+
events: &[Event<'_>],
|
|
106
|
+
params: &TruncateParams,
|
|
107
|
+
md_opts: pulldown_cmark::Options,
|
|
108
|
+
) -> Result<String, SerializeError> {
|
|
96
109
|
let marker_chars = marker_chars(¶ms.marker);
|
|
97
110
|
let marker_words = marker_words(¶ms.marker);
|
|
98
111
|
let char_budget = params.chars.map(|n| n.saturating_sub(marker_chars));
|
|
@@ -104,8 +117,14 @@ fn truncate_at_block(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
104
117
|
let mut any_dropped = false;
|
|
105
118
|
|
|
106
119
|
for (start, end) in top_level_blocks(events) {
|
|
107
|
-
let block =
|
|
108
|
-
|
|
120
|
+
let block = markdown::render_block(&events[start..=end], md_opts)?;
|
|
121
|
+
// The blank line rejoining blocks counts toward the char budget.
|
|
122
|
+
let separator = if kept.is_empty() {
|
|
123
|
+
""
|
|
124
|
+
} else {
|
|
125
|
+
markdown::BLOCK_SEPARATOR
|
|
126
|
+
};
|
|
127
|
+
let block_chars = separator.len() + block.trim_end().chars().count();
|
|
109
128
|
let block_words = block.unicode_words().count();
|
|
110
129
|
|
|
111
130
|
let would_exceed_chars = char_budget
|
|
@@ -120,6 +139,7 @@ fn truncate_at_block(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
120
139
|
break;
|
|
121
140
|
}
|
|
122
141
|
|
|
142
|
+
kept.push_str(separator);
|
|
123
143
|
kept.push_str(&block);
|
|
124
144
|
used_chars += block_chars;
|
|
125
145
|
used_words += block_words;
|
|
@@ -127,12 +147,12 @@ fn truncate_at_block(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
127
147
|
|
|
128
148
|
// If nothing was dropped and source fit, return unchanged (no marker).
|
|
129
149
|
if !any_dropped {
|
|
130
|
-
return kept;
|
|
150
|
+
return Ok(kept);
|
|
131
151
|
}
|
|
132
152
|
|
|
133
153
|
// First block alone exceeded the budget; honest empty return
|
|
134
154
|
if kept.is_empty() {
|
|
135
|
-
return String::new();
|
|
155
|
+
return Ok(String::new());
|
|
136
156
|
}
|
|
137
157
|
|
|
138
158
|
// Trim trailing whitespace before appending the marker so the
|
|
@@ -145,7 +165,7 @@ fn truncate_at_block(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
145
165
|
kept.push_str(marker);
|
|
146
166
|
}
|
|
147
167
|
kept.push('\n');
|
|
148
|
-
kept
|
|
168
|
+
Ok(kept)
|
|
149
169
|
}
|
|
150
170
|
|
|
151
171
|
/// Return (start, end) event-index pairs for top-level blocks.
|
|
@@ -185,15 +205,12 @@ fn top_level_blocks(events: &[Event<'_>]) -> Vec<(usize, usize)> {
|
|
|
185
205
|
blocks
|
|
186
206
|
}
|
|
187
207
|
|
|
188
|
-
fn
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
fn truncate_at_word(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
196
|
-
let rendered = render_markdown(events);
|
|
208
|
+
fn truncate_at_word(
|
|
209
|
+
events: &[Event<'_>],
|
|
210
|
+
params: &TruncateParams,
|
|
211
|
+
md_opts: pulldown_cmark::Options,
|
|
212
|
+
) -> Result<String, SerializeError> {
|
|
213
|
+
let rendered = markdown::render(events, md_opts)?;
|
|
197
214
|
let marker_chars = marker_chars(¶ms.marker);
|
|
198
215
|
let marker_words = marker_words(¶ms.marker);
|
|
199
216
|
|
|
@@ -211,7 +228,7 @@ fn truncate_at_word(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
211
228
|
|
|
212
229
|
// Fits under both budgets: return unchanged, no marker.
|
|
213
230
|
if chars_ok && words_ok {
|
|
214
|
-
return rendered;
|
|
231
|
+
return Ok(rendered);
|
|
215
232
|
}
|
|
216
233
|
|
|
217
234
|
let char_budget = params.chars.map(|n| n.saturating_sub(marker_chars));
|
|
@@ -252,7 +269,7 @@ fn truncate_at_word(events: &[Event<'_>], params: &TruncateParams) -> String {
|
|
|
252
269
|
if let Some(marker) = ¶ms.marker {
|
|
253
270
|
out.push_str(marker);
|
|
254
271
|
}
|
|
255
|
-
out
|
|
272
|
+
Ok(out)
|
|
256
273
|
}
|
|
257
274
|
|
|
258
275
|
fn marker_chars(marker: &Option<String>) -> usize {
|
data/lib/inkmark/version.rb
CHANGED