inkmark 0.2.1 → 0.3.1

data/ext/inkmark/src/chunks_by_size.rs

@@ -22,6 +22,7 @@ use pulldown_cmark::Event;
 use unicode_segmentation::UnicodeSegmentation;
 
 use crate::document::{apply_filters, content_events};
+use crate::markdown::{self, SerializeError};
 use crate::options::build_options;
 
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -48,10 +49,12 @@ pub fn native_chunks_by_size(
     let events: Vec<Event> = content_events(&source, cm_opts).collect();
     let events = apply_filters(events, &flags);
 
+    let md_opts = markdown::serializer_options(cm_opts, &flags);
     let windows = match params.at {
-        BoundaryAt::Block => chunk_blocks(&events, &params),
-        BoundaryAt::Word => chunk_words(&events, &params),
-    };
+        BoundaryAt::Block => chunk_blocks(&events, &params, md_opts),
+        BoundaryAt::Word => chunk_words(&events, &params, md_opts),
+    }
+    .map_err(|e| markdown::to_ruby_error(ruby, e))?;
 
     build_result(ruby, &windows, flags.statistics)
 }
@@ -79,20 +82,30 @@ fn parse_params(ruby: &Ruby, hash: &RHash) -> Result<WindowParams, Error> {
     })
 }
 
-fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
+fn chunk_blocks(
+    events: &[Event<'_>],
+    params: &WindowParams,
+    md_opts: pulldown_cmark::Options,
+) -> Result<Vec<String>, SerializeError> {
     let mut out: Vec<String> = Vec::new();
     let mut current = String::new();
     let mut current_chars: usize = 0;
     let mut current_words: usize = 0;
 
     for (start, end) in top_level_blocks(events) {
-        let block = render_markdown(&events[start..=end]);
+        let block = markdown::render_block(&events[start..=end], md_opts)?;
         let block_chars = block.chars().count();
         let block_words = block.unicode_words().count();
+        // The blank line rejoining blocks counts toward the char budget.
+        let separator_chars = if current.is_empty() {
+            0
+        } else {
+            markdown::BLOCK_SEPARATOR.len()
+        };
 
         let would_exceed_chars = params
             .chars
-            .map(|b| current_chars + block_chars > b)
+            .map(|b| current_chars + separator_chars + block_chars > b)
             .unwrap_or(false);
         let would_exceed_words = params
             .words
@@ -110,6 +123,11 @@ fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
             out.push(finished);
         }
 
+        // Re-checked: a flush leaves `current` holding the overlap seed.
+        if !current.is_empty() {
+            current.push_str(markdown::BLOCK_SEPARATOR);
+            current_chars += markdown::BLOCK_SEPARATOR.len();
+        }
         current.push_str(&block);
         current_chars += block_chars;
         current_words += block_words;
@@ -118,7 +136,7 @@ fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
     if !current.is_empty() {
         out.push(current);
     }
-    out
+    Ok(out)
 }
 
 /// Return the trailing `overlap` characters of `s`, aligned to a
@@ -167,13 +185,6 @@ fn top_level_blocks(events: &[Event<'_>]) -> Vec<(usize, usize)> {
     blocks
 }
 
-fn render_markdown(events: &[Event<'_>]) -> String {
-    let mut buf = String::new();
-    pulldown_cmark_to_cmark::cmark(events.iter().cloned(), &mut buf)
-        .expect("markdown serialization failed");
-    buf
-}
-
 // Serialize the full filtered Markdown into one string, then walk
 // word boundaries. Each window is a byte-aligned slice of the
 // serialized output ending at a word boundary that fits the budget.
@@ -181,10 +192,14 @@ fn render_markdown(events: &[Event<'_>]) -> String {
 // `overlap` chars (to the next word boundary) before taking the
 // next window.
 
-fn chunk_words(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
-    let rendered = render_markdown(events);
+fn chunk_words(
+    events: &[Event<'_>],
+    params: &WindowParams,
+    md_opts: pulldown_cmark::Options,
+) -> Result<Vec<String>, SerializeError> {
+    let rendered = markdown::render(events, md_opts)?;
     if rendered.is_empty() {
-        return Vec::new();
+        return Ok(Vec::new());
     }
 
     let mut out: Vec<String> = Vec::new();
@@ -250,7 +265,7 @@ fn chunk_words(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
         };
     }
 
-    out
+    Ok(out)
 }
 
 /// Step the cursor back by roughly `overlap` characters, then land on

data/ext/inkmark/src/document.rs

@@ -7,6 +7,7 @@ use crate::heading;
 use crate::highlight;
 use crate::image;
 use crate::link;
+use crate::markdown::{self, SerializeError};
 use crate::options::{build_options, Flags};
 use crate::plain_text;
 use crate::scheme_filter::SchemeFilter;
@@ -36,13 +37,14 @@ pub fn native_to_markdown(
     source: String,
     opts_hash: Option<RHash>,
 ) -> Result<String, Error> {
-    match opts_hash {
-        None => Ok(markdown_defaults(&source)),
+    let rendered = match opts_hash {
+        None => markdown_defaults(&source),
         Some(hash) => {
             let (cm_opts, flags) = build_options(ruby, hash)?;
-            Ok(render_to_markdown(&source, cm_opts, flags))
+            render_to_markdown(&source, cm_opts, flags)
         }
-    }
+    };
+    rendered.map_err(|e| markdown::to_ruby_error(ruby, e))
 }
 
 pub fn native_to_plain_text(
@@ -71,12 +73,13 @@ fn render_defaults(source: &str) -> String {
 }
 
 /// Same as render_defaults but serializes to Markdown instead of HTML.
-fn markdown_defaults(source: &str) -> String {
+fn markdown_defaults(source: &str) -> Result<String, SerializeError> {
     let mut buf = String::with_capacity(source.len());
-    let parser = Parser::new_ext(source, default_cm_opts());
+    let cm_opts = default_cm_opts();
+    let parser = Parser::new_ext(source, cm_opts);
     let filtered = parser.map(suppress_raw_html);
-    cmark_write(filtered, &mut buf);
-    buf
+    markdown::write(filtered, &mut buf, cm_opts)?;
+    Ok(buf)
 }
 
 /// Defaults-only plain-text fast path. Mirrors `markdown_defaults`:
@@ -255,22 +258,25 @@ fn render(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> Strin
     buf
 }
 
-fn render_to_markdown(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> String {
-    let mut buf = String::with_capacity(source.len());
-
+fn render_to_markdown(
+    source: &str,
+    cm_opts: pulldown_cmark::Options,
+    flags: Flags,
+) -> Result<String, SerializeError> {
     // `content_events` strips frontmatter, so the cmark serializer never sees
     // a metadata block to re-emit—on either the streaming or buffered path.
     if !needs_buffer(&flags) {
-        cmark_write(
+        let mut buf = String::with_capacity(source.len());
+        markdown::write(
             content_events(source, cm_opts).map(stream_filter(&flags)),
             &mut buf,
-        );
-        return buf;
+            cm_opts,
+        )?;
+        return Ok(buf);
     }
 
     let events = apply_filters(content_events(source, cm_opts).collect(), &flags);
-    cmark_write(events.into_iter(), &mut buf);
-    buf
+    markdown::render(&events, markdown::serializer_options(cm_opts, &flags))
 }
 
 fn render_to_plain_text(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> String {
@@ -447,7 +453,3 @@ pub fn apply_post_handler_filters<'a>(mut events: Vec<Event<'a>>, flags: &Flags)
 
     events
 }
-
-fn cmark_write<'a, I: Iterator<Item = Event<'a>>>(events: I, buf: &mut String) {
-    pulldown_cmark_to_cmark::cmark(events, buf).expect("markdown serialization failed");
-}

data/ext/inkmark/src/image.rs

@@ -1,7 +1,7 @@
 //! Image attribute injection filter and URL matcher.
 //!
 //! When enabled, replaces pulldown-cmark's default image event sequence
-//! (`Start(Tag::Image) ... End(TagEnd::Image)`) with a single `Event::Html`
+//! (`Start(Tag::Image) ... End(TagEnd::Image)`) with a single `Event::InlineHtml`
 //! carrying a hand-built `<img>` tag that includes the "modern" loading and
 //! decoding hints:
 //!
@@ -23,7 +23,7 @@ use pulldown_cmark_escape::{escape_href, escape_html};
 
 use crate::url_match::is_host_allowed;
 
-/// Rewrite every image in the event stream as a self-contained `Event::Html`
+/// Rewrite every image in the event stream as a self-contained `Event::InlineHtml`
 /// carrying `<img ... loading="lazy" decoding="async">`.
 ///
 /// We consume the input Vec to own each event, then rebuild with
@@ -38,22 +38,9 @@ pub fn add_lazy_loading(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
             Event::Start(Tag::Image {
                 dest_url, title, ..
             }) => {
-                // Consume events up to the matching End(Image), accumulating
-                // alt text from Text and Code payloads. Images can contain
-                // inline formatting (e.g. `![**bold**](img.png)`), which
-                // produces Start(Strong)/Text/End(Strong) events; the bare
-                // text content is what we want for the alt attribute.
-                let mut alt = String::new();
-                for inner in iter.by_ref() {
-                    match inner {
-                        Event::End(TagEnd::Image) => break,
-                        Event::Text(t) | Event::Code(t) => alt.push_str(&t),
-                        _ => {}
-                    }
-                }
-
+                let alt = take_alt_text(&mut iter);
                 let html = build_img_tag(&dest_url, &alt, &title);
-                out.push(Event::Html(CowStr::Boxed(html.into_boxed_str())));
+                out.push(Event::InlineHtml(CowStr::Boxed(html.into_boxed_str())));
             }
             other => out.push(other),
         }
@@ -68,10 +55,8 @@ pub fn add_lazy_loading(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
 /// when alt is empty. Non-web URLs pass through: [`is_host_allowed`]
 /// returns true for any URL with no parseable host.
 ///
-/// Alt accumulation matches `add_lazy_loading`: images can contain
-/// markdown like `![**bold**](img.png)`, producing
-/// Start(Strong)/Text/End(Strong) events—we pull the raw text payloads
-/// out and discard formatting.
+/// Alt accumulation is shared with `add_lazy_loading` (see
+/// [`take_alt_text`]).
 pub fn filter_by_hosts<'a>(events: Vec<Event<'a>>, set: &GlobSet) -> Vec<Event<'a>> {
     let mut out: Vec<Event<'a>> = Vec::with_capacity(events.len());
     let mut iter = events.into_iter();
@@ -79,14 +64,7 @@ pub fn filter_by_hosts<'a>(events: Vec<Event<'a>>, set: &GlobSet) -> Vec<Event<'
     while let Some(event) = iter.next() {
         match event {
             Event::Start(Tag::Image { ref dest_url, .. }) if !is_host_allowed(dest_url, set) => {
-                let mut alt = String::new();
-                for inner in iter.by_ref() {
-                    match inner {
-                        Event::End(TagEnd::Image) => break,
-                        Event::Text(t) | Event::Code(t) => alt.push_str(&t),
-                        _ => {}
-                    }
-                }
+                let alt = take_alt_text(&mut iter);
                 if !alt.is_empty() {
                     out.push(Event::Text(CowStr::Boxed(alt.into_boxed_str())));
                 }
@@ -98,6 +76,30 @@ pub fn filter_by_hosts<'a>(events: Vec<Event<'a>>, set: &GlobSet) -> Vec<Event<'
     out
 }
 
+/// Consume events through the `End(Image)` matching an already-consumed
+/// `Start(Image)` and return the alt text: the raw Text and Code payloads,
+/// with formatting (`![**bold**](img.png)`) discarded. An image nested in
+/// the alt text (`![a ![b](x)](y)`) contributes its text too, so depth is
+/// counted to stop at the matching End rather than the nested image's.
+fn take_alt_text<'a>(iter: &mut impl Iterator<Item = Event<'a>>) -> String {
+    let mut alt = String::new();
+    let mut depth: usize = 1;
+    for inner in iter {
+        match inner {
+            Event::Start(Tag::Image { .. }) => depth += 1,
+            Event::End(TagEnd::Image) => {
+                depth -= 1;
+                if depth == 0 {
+                    break;
+                }
+            }
+            Event::Text(t) | Event::Code(t) => alt.push_str(&t),
+            _ => {}
+        }
+    }
+    alt
+}
+
 /// Construct the `<img>` HTML string with `loading="lazy"` and
 /// `decoding="async"` attributes. `src` is escaped as a URL (percent-
 /// encoded where necessary); `alt` and `title` are HTML-attribute
@@ -201,7 +203,7 @@ mod tests {
         let out = add_lazy_loading(events);
         assert_eq!(out.len(), 1, "should collapse to one event");
         match &out[0] {
-            Event::Html(html) => {
+            Event::InlineHtml(html) => {
                 assert!(
                     html.contains("loading="),
                     "missing loading attribute: {html}"
@@ -233,6 +235,31 @@ mod tests {
         }
     }
 
+    fn has_image_event(events: &[Event<'_>]) -> bool {
+        events.iter().any(|e| {
+            matches!(
+                e,
+                Event::Start(Tag::Image { .. }) | Event::End(TagEnd::Image)
+            )
+        })
+    }
+
+    #[test]
+    fn image_nested_in_alt_text_is_consumed_with_its_parent() {
+        let source = "![foo ![bar](https://evil.com/1)](https://evil.com/2)";
+        let events: Vec<Event> = pulldown_cmark::Parser::new(source).collect();
+
+        let dropped = filter_by_hosts(events.clone(), &host_set(&["example.net"]));
+        assert!(!has_image_event(&dropped), "stray image event: {dropped:?}");
+        assert!(dropped.contains(&Event::Text(CowStr::Borrowed("foo bar"))));
+
+        let lazy = add_lazy_loading(events);
+        assert!(!has_image_event(&lazy), "stray image event: {lazy:?}");
+        assert!(lazy
+            .iter()
+            .any(|e| matches!(e, Event::InlineHtml(h) if h.contains("alt=\"foo bar\""))));
+    }
+
     #[test]
     fn filter_by_hosts_drops_disallowed_image_with_empty_alt_entirely() {
         let events = vec![

data/ext/inkmark/src/lib.rs

@@ -14,6 +14,7 @@ mod heading;
 mod highlight;
 mod image;
 mod link;
+mod markdown;
 mod options;
 mod plain_text;
 mod scheme_filter;

data/ext/inkmark/src/link.rs

@@ -18,8 +18,9 @@ use pulldown_cmark_escape::{escape_href, escape_html};
 use crate::url_match::is_host_allowed;
 
 /// Add `rel="nofollow noopener"` to every external `<a>` tag by replacing
-/// its `Start(Link)` event with a synthesized `Event::Html` opening tag
-/// and its matching `End(Link)` event with a `</a>` close tag.
+/// its `Start(Link)` event with a synthesized `Event::InlineHtml` opening
+/// tag and its matching `End(Link)` event with a `</a>` close tag. Inline,
+/// not block, HTML: the Markdown serializer needs to know it sits in a line.
 pub fn add_nofollow(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
     let mut out: Vec<Event<'_>> = Vec::with_capacity(events.len());
     let mut iter = events.into_iter();
@@ -33,7 +34,7 @@ pub fn add_nofollow(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
                 id: _,
             }) if is_external(dest_url) => {
                 let open = build_link_open(dest_url, title);
-                out.push(Event::Html(CowStr::Boxed(open.into_boxed_str())));
+                out.push(Event::InlineHtml(CowStr::Boxed(open.into_boxed_str())));
 
                 // Consume inner events through the matching End(Link),
                 // depth-counting so a nested link doesn't break the
@@ -51,7 +52,7 @@ pub fn add_nofollow(events: Vec<Event<'_>>) -> Vec<Event<'_>> {
                     } else if is_link_end {
                         depth -= 1;
                         if depth == 0 {
-                            out.push(Event::Html(CowStr::Borrowed("</a>")));
+                            out.push(Event::InlineHtml(CowStr::Borrowed("</a>")));
                             break;
                         }
                         out.push(inner);
@@ -209,10 +210,10 @@ mod tests {
             Event::End(TagEnd::Link),
         ];
         let out = add_nofollow(events);
-        // Should produce: Html(open), Text("click"), Html("</a>")
+        // Should produce: InlineHtml(open), Text("click"), InlineHtml("</a>")
         assert_eq!(out.len(), 3, "expected 3 events, got {}", out.len());
         match &out[0] {
-            Event::Html(html) => {
+            Event::InlineHtml(html) => {
                 assert!(
                     html.contains("nofollow"),
                     "opening tag must contain nofollow: {html}"
@@ -222,11 +223,11 @@ mod tests {
                     "opening tag must contain href: {html}"
                 );
             }
-            other => panic!("expected Html open event, got {other:?}"),
+            other => panic!("expected InlineHtml open event, got {other:?}"),
         }
         match &out[2] {
-            Event::Html(html) => assert_eq!(html.as_ref(), "</a>"),
-            other => panic!("expected Html close event, got {other:?}"),
+            Event::InlineHtml(html) => assert_eq!(html.as_ref(), "</a>"),
+            other => panic!("expected InlineHtml close event, got {other:?}"),
         }
     }