inkmark 0.2.2 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99be70c153a0446a98b513e0546389622cb5815d5907388f0b9160da7b1652cc
-  data.tar.gz: b156f0536a351ef5780816a10ccedbb4d5a5a3a6c8741f2ccbbd065247f249ed
+  metadata.gz: 4a3feea6c75cbbc48daa868ce5aa140f1929df7b6b109ee83e6ed67c4b290351
+  data.tar.gz: ddddb3918c21fe25cf33f2b58370475c9e5f4a1a0b2963b32de904d9d096cf1f
 SHA512:
-  metadata.gz: b7dcd628ed44d5c5ab757f257c95f2b23ad6880835f68db65f735ba45a6f7666de1bff0be3cc6e088ba18f3407848db0cc9c63d970e1a415bba1451ac58a15d7
-  data.tar.gz: b704a73b47b7e85db26b13b1c02dc5a840939fea7eed13b9f4e7b68f816a23add8ed1bc60d3278656d3f4951dad318519ac14d257286ea513abbcf55dc7d622e
+  metadata.gz: 0a02d6c1b1f9067787cf880fbf044b59bc05220bc075d99332af7e0cac416f2bdcf2545b935d7d309db45411d67f01ab4f0614ed52f0ddc1f28bb261ddff892c
+  data.tar.gz: 242ecf84608b2de0fdd37e883506d3fbd0ec7cb1dc5184bbb707e1648a99e0ca4d3834ee48d74b21d69d90cf5c3922fb1c36d48754b521c858c20e00c3af9e17

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## [0.3.1] - 2026-10-24
+
+- Re-release of 0.3.0, which was yanked.
+- Breaking: Markdown output (`to_markdown`, `chunks_by_heading`, `chunks_by_size`, `truncate_markdown`) uses pulldown-cmark-to-cmark 23; escaping and formatting differ from 0.2.
+- Fix raw HTML leaking into Markdown output with `raw_html: false` when a line held more than one tag.
+- Fix `chunks_by_size` and `truncate_markdown` (`at: :block`) joining blocks without a blank line, which merged adjacent paragraphs.
+- Fix Markdown output crashing on an image nested in another image's alt text with `images: { lazy: true }`, `allowed_hosts`, or `allowed_schemes`.
+- `chunks_by_heading` starts sections only at top-level headings; a heading inside a block quote or list item no longer splits it.
+- Raise `Inkmark::Error` on Markdown serialization failures instead of crashing the process.
+
+## [0.3.0] - 2026-10-24 [YANKED]
+
+- Yanked: a RubyGems issue interrupted publishing, so only one platform gem was released. Its changes shipped in 0.3.1.
+
 ## [0.2.2] - 2026-10-24
 
 - Updated dependencies, including pulldown-cmark 0.13.5 (upstream security fixes) and magnus 0.9.3.

data/Cargo.lock

@@ -333,7 +333,7 @@ dependencies = [
 
 [[package]]
 name = "inkmark"
-version = "0.2.2"
+version = "0.3.1"
 dependencies = [
  "deunicode",
  "emojis",
@@ -527,11 +527,12 @@ checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
 
 [[package]]
 name = "pulldown-cmark-to-cmark"
-version = "22.0.3"
+version = "23.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867"
+checksum = "e9e14cd4ec725b2227ba76d4772169af814438bcf48f6d9b806d1308f40a7413"
 dependencies = [
  "pulldown-cmark",
+ "unicase",
 ]
 
 [[package]]

data/README.md

@@ -206,6 +206,9 @@ Inkmark.to_html("<script>alert(1)</script>")
 # => "<p>&lt;script&gt;alert(1)&lt;/script&gt;</p>\n"
 ```
 
+Markdown output escapes it too; see
+[Raw HTML in the source](#raw-html-in-the-source).
+
 Enable pass-through with `raw_html: true`; _only do this for trusted
 input_:
 
@@ -403,6 +406,12 @@ nested `###` subsections, which also appear as their own entries. Content
 before the first heading (if any) becomes a preamble entry with
 `heading: nil` and `level: 0`.
 
+Only **top-level** headings start a section. A heading inside a block
+quote or list item (such as a `## Tip` in a `> [!NOTE]` callout) stays in
+the content of the section around it, so the quote or list is never split
+across sections. It still counts toward slug deduplication, so every
+section's `:id` matches the anchor `headings: { ids: true }` generates.
+
 `:breadcrumb` carries the ancestor heading texts from root to immediate
 parent. Root-level sections and the preamble have an empty array. Skipped
 levels are omitted, so an `###` directly under an `#` has `breadcrumb:
@@ -740,6 +749,17 @@ Inkmark.to_markdown(source, options: {
 })
 ```
 
+### Raw HTML in the source
+
+With `raw_html: false` (the default), raw HTML from the source comes out of
+`to_markdown`, `chunks_by_heading`, `chunks_by_size`, and `truncate_markdown`
+escaped, so the Markdown stays inert even if a later stage renders it with raw
+HTML allowed. HTML produced by the HTML-emitting filters above is the
+exception: it is embedded as described.
+
+If the filtered document cannot be written back as Markdown, these methods
+raise `Inkmark::Error`.
+
 ## Event handlers
 
 Register handlers with `#on` to inspect or transform document elements as they

data/ext/inkmark/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "inkmark"
-version = "0.2.2"
+version = "0.3.1"
 edition = "2021"
 authors = ["Yaroslav Markin <yaroslav@markin.net>"]
 license = "MIT"
@@ -22,7 +22,7 @@ syntect = { version = "5.2", default-features = false, features = ["parsing", "h
 linkify = "0.11"
 globset = "0.4"
 url = "2"
-pulldown-cmark-to-cmark = "22"
+pulldown-cmark-to-cmark = "23"
 
 [build-dependencies]
 rb-sys-env = "0.2.2"

data/ext/inkmark/src/chunks_by_heading.rs

@@ -14,6 +14,12 @@
 //! emoji/autolink rewrites happen *inside* a heading's text events,
 //! but the bracketing tags stay in place—so post-filter heading-
 //! position scanning is coherent.
+//!
+//! Only top-level headings start a section. A heading inside a block
+//! quote, list item, or other container stays in the content of the
+//! section around it, so a container is never split across sections.
+//! Nested headings still consume slugs, keeping section ids aligned
+//! with the ids `heading_ids` / `toc` emit.
 
 use magnus::{Error, RArray, RHash, Ruby};
 use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
@@ -21,6 +27,7 @@ use unicode_segmentation::UnicodeSegmentation;
 
 use crate::document::{apply_filters, content_events};
 use crate::heading::{self, SlugDeduplicator};
+use crate::markdown;
 use crate::options::build_options;
 use crate::toc;
 use crate::truncate::{self, TruncateParams};
@@ -57,8 +64,13 @@ pub fn native_chunks_by_heading(
     let content_opts = ContentOptions {
         with_counts: flags.statistics,
         truncate_params: truncate_params.as_ref(),
+        md_opts: markdown::serializer_options(cm_opts, &flags),
     };
-    let preamble_end = boundaries.first().map(|b| b.start).unwrap_or(events.len());
+    let preamble_end = boundaries
+        .iter()
+        .find(|b| b.top_level)
+        .map(|b| b.start)
+        .unwrap_or(events.len());
     if preamble_end > 0 {
         let preamble_events = &events[0..preamble_end];
         if !is_empty_content(preamble_events) {
@@ -78,23 +90,29 @@ pub fn native_chunks_by_heading(
     let mut dedup = SlugDeduplicator::new();
     let mut ancestors: Vec<(u8, String)> = Vec::new();
     for (i, boundary) in boundaries.iter().enumerate() {
+        let heading_text = collect_inline_text(&events[(boundary.start + 1)..boundary.end]);
+        let id = heading_id(&mut dedup, &heading_text);
+        if !boundary.top_level {
+            // Stays in the surrounding section's content; only its slug
+            // counts, so later ids match `heading_ids`.
+            continue;
+        }
         let section_end = find_section_end(&boundaries, i, events.len());
         let level = toc::level_to_u8(boundary.level);
         while ancestors.last().is_some_and(|(l, _)| *l >= level) {
             ancestors.pop();
         }
         let breadcrumb: Vec<&str> = ancestors.iter().map(|(_, t)| t.as_str()).collect();
-        let heading_text = collect_inline_text(&events[(boundary.start + 1)..boundary.end]);
         // Content = events after End(Heading) up to the next section or
-        // end of document. Re-serialized through cmark_write.
+        // end of document. Re-serialized through `markdown::render`.
         let content_events = &events[(boundary.end + 1)..section_end];
         let hash = build_section_hash(
             ruby,
             boundary.level,
             &heading_text,
+            &id,
             &breadcrumb,
             content_events,
-            &mut dedup,
             &content_opts,
         )?;
         result.push(hash)?;
@@ -110,6 +128,9 @@ struct HeadingBoundary {
     start: usize,
     end: usize,
     level: HeadingLevel,
+    /// Not inside a block quote, list item, or other container. Only
+    /// top-level headings start sections.
+    top_level: bool,
 }
 
 /// How each entry's `content` is rendered: optionally truncated, and
@@ -117,14 +138,20 @@ struct HeadingBoundary {
 struct ContentOptions<'a> {
     with_counts: bool,
     truncate_params: Option<&'a TruncateParams>,
+    /// Serializer options, from `markdown::serializer_options`.
+    md_opts: pulldown_cmark::Options,
 }
 
 fn find_heading_boundaries(events: &[Event<'_>]) -> Vec<HeadingBoundary> {
     let mut boundaries = Vec::new();
+    // Containers open around the cursor. Headings are leaf blocks, so
+    // none open at a Start(Heading) means it is not inside anything.
+    let mut open_containers = 0usize;
     let mut i = 0;
     while i < events.len() {
         if let Event::Start(Tag::Heading { level, .. }) = &events[i] {
             let lvl = *level;
+            let top_level = open_containers == 0;
             // CommonMark disallows headings inside headings, but carry a
             // depth counter so we stay correct if pulldown-cmark ever
             // permits them.
@@ -147,23 +174,29 @@ fn find_heading_boundaries(events: &[Event<'_>]) -> Vec<HeadingBoundary> {
                 start: i,
                 end: j,
                 level: lvl,
+                top_level,
             });
             i = j + 1;
         } else {
+            match &events[i] {
+                Event::Start(_) => open_containers += 1,
+                Event::End(_) => open_containers = open_containers.saturating_sub(1),
+                _ => {}
+            }
             i += 1;
         }
     }
     boundaries
 }
 
-/// Section i ends at the first subsequent heading with level <=
-/// current. Headings with a strictly greater level are subsections:
+/// Section i ends at the first subsequent top-level heading with level
+/// <= current. Headings with a strictly greater level are subsections:
 /// they belong to the current section's content too.
 fn find_section_end(boundaries: &[HeadingBoundary], i: usize, events_len: usize) -> usize {
     let current = toc::level_to_u8(boundaries[i].level);
     boundaries[(i + 1)..]
         .iter()
-        .find(|b| toc::level_to_u8(b.level) <= current)
+        .find(|b| b.top_level && toc::level_to_u8(b.level) <= current)
         .map(|b| b.start)
         .unwrap_or(events_len)
 }
@@ -181,10 +214,7 @@ fn build_preamble_hash(
     // with proper sections so callers can treat every entry alike.
     hash.aset(ruby.to_symbol("breadcrumb"), ruby.ary_new_capa(0))?;
 
-    let content = match content_opts.truncate_params {
-        Some(params) => truncate::truncate_events(events, params),
-        None => render_markdown(events),
-    };
+    let content = render_content(ruby, events, content_opts)?;
     if content_opts.with_counts {
         let (chars, words) = count_post_truncate(events, content_opts.truncate_params, &content);
         hash.aset(ruby.to_symbol("character_count"), chars)?;
@@ -198,21 +228,11 @@ fn build_section_hash(
     ruby: &Ruby,
     level: HeadingLevel,
     heading_text: &str,
+    id: &str,
     breadcrumb: &[&str],
     content_events: &[Event<'_>],
-    dedup: &mut SlugDeduplicator,
     content_opts: &ContentOptions<'_>,
 ) -> Result<RHash, Error> {
-    // Slug is the deduplicated slugify of the (filter-applied) heading
-    // text, matching the ids `heading_ids` / `toc` would emit for the
-    // same document.
-    let base = heading::slugify(heading_text);
-    let id = if base.is_empty() {
-        String::new()
-    } else {
-        dedup.deduplicate(base)
-    };
-
     let hash = ruby.hash_new();
     hash.aset(ruby.to_symbol("heading"), heading_text)?;
     hash.aset(ruby.to_symbol("level"), toc::level_to_u8(level))?;
@@ -227,10 +247,7 @@ fn build_section_hash(
     }
     hash.aset(ruby.to_symbol("breadcrumb"), breadcrumb_arr)?;
 
-    let content = match content_opts.truncate_params {
-        Some(params) => truncate::truncate_events(content_events, params),
-        None => render_markdown(content_events),
-    };
+    let content = render_content(ruby, content_events, content_opts)?;
     if content_opts.with_counts {
         let (chars, words) =
             count_post_truncate(content_events, content_opts.truncate_params, &content);
@@ -289,11 +306,24 @@ fn count_text(events: &[Event<'_>]) -> (usize, usize) {
     (chars, words)
 }
 
-fn render_markdown(events: &[Event<'_>]) -> String {
-    let mut buf = String::new();
-    pulldown_cmark_to_cmark::cmark(events.iter().cloned(), &mut buf)
-        .expect("markdown serialization failed");
-    buf
+/// Deduplicated slug of the (filter-applied) heading text, matching the
+/// ids `heading_ids` / `toc` would emit for the same document. Empty when
+/// the text has nothing to slugify.
+fn heading_id(dedup: &mut SlugDeduplicator, heading_text: &str) -> String {
+    dedup.deduplicate(heading::slugify(heading_text))
+}
+
+/// Serialize a section's events to Markdown, truncated when requested.
+fn render_content(
+    ruby: &Ruby,
+    events: &[Event<'_>],
+    content_opts: &ContentOptions<'_>,
+) -> Result<String, Error> {
+    match content_opts.truncate_params {
+        Some(params) => truncate::truncate_events(events, params, content_opts.md_opts),
+        None => markdown::render(events, content_opts.md_opts),
+    }
+    .map_err(|e| markdown::to_ruby_error(ruby, e))
 }
 
 fn collect_inline_text(events: &[Event<'_>]) -> String {
@@ -324,3 +354,33 @@ fn is_empty_content(events: &[Event<'_>]) -> bool {
         )
     })
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pulldown_cmark::Options;
+
+    fn events(source: &str) -> Vec<Event<'_>> {
+        Parser::new_ext(source, Options::empty()).collect()
+    }
+
+    #[test]
+    fn headings_inside_containers_are_not_top_level() {
+        let ev = events("# Top\n\n> ## Quoted\n\n- # Listed\n\n## Sub\n");
+        let top_level: Vec<bool> = find_heading_boundaries(&ev)
+            .iter()
+            .map(|b| b.top_level)
+            .collect();
+        assert_eq!(top_level, vec![true, false, false, true]);
+    }
+
+    #[test]
+    fn section_spanning_a_nested_heading_stays_balanced() {
+        let ev = events("# Top\n\n> ## Inner\n> text\n\nafter\n");
+        let boundaries = find_heading_boundaries(&ev);
+        let end = find_section_end(&boundaries, 0, ev.len());
+        let content = &ev[(boundaries[0].end + 1)..end];
+        let md = markdown::render(content, Options::empty()).expect("balanced slice");
+        assert!(md.contains("Inner") && md.contains("after"), "{md:?}");
+    }
+}

data/ext/inkmark/src/chunks_by_size.rs

@@ -22,6 +22,7 @@ use pulldown_cmark::Event;
 use unicode_segmentation::UnicodeSegmentation;
 
 use crate::document::{apply_filters, content_events};
+use crate::markdown::{self, SerializeError};
 use crate::options::build_options;
 
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -48,10 +49,12 @@ pub fn native_chunks_by_size(
     let events: Vec<Event> = content_events(&source, cm_opts).collect();
     let events = apply_filters(events, &flags);
 
+    let md_opts = markdown::serializer_options(cm_opts, &flags);
     let windows = match params.at {
-        BoundaryAt::Block => chunk_blocks(&events, &params),
-        BoundaryAt::Word => chunk_words(&events, &params),
-    };
+        BoundaryAt::Block => chunk_blocks(&events, &params, md_opts),
+        BoundaryAt::Word => chunk_words(&events, &params, md_opts),
+    }
+    .map_err(|e| markdown::to_ruby_error(ruby, e))?;
 
     build_result(ruby, &windows, flags.statistics)
 }
@@ -79,20 +82,30 @@ fn parse_params(ruby: &Ruby, hash: &RHash) -> Result<WindowParams, Error> {
     })
 }
 
-fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
+fn chunk_blocks(
+    events: &[Event<'_>],
+    params: &WindowParams,
+    md_opts: pulldown_cmark::Options,
+) -> Result<Vec<String>, SerializeError> {
     let mut out: Vec<String> = Vec::new();
     let mut current = String::new();
     let mut current_chars: usize = 0;
     let mut current_words: usize = 0;
 
     for (start, end) in top_level_blocks(events) {
-        let block = render_markdown(&events[start..=end]);
+        let block = markdown::render_block(&events[start..=end], md_opts)?;
         let block_chars = block.chars().count();
         let block_words = block.unicode_words().count();
+        // The blank line rejoining blocks counts toward the char budget.
+        let separator_chars = if current.is_empty() {
+            0
+        } else {
+            markdown::BLOCK_SEPARATOR.len()
+        };
 
         let would_exceed_chars = params
             .chars
-            .map(|b| current_chars + block_chars > b)
+            .map(|b| current_chars + separator_chars + block_chars > b)
             .unwrap_or(false);
         let would_exceed_words = params
             .words
@@ -110,6 +123,11 @@ fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
             out.push(finished);
         }
 
+        // Re-checked: a flush leaves `current` holding the overlap seed.
+        if !current.is_empty() {
+            current.push_str(markdown::BLOCK_SEPARATOR);
+            current_chars += markdown::BLOCK_SEPARATOR.len();
+        }
         current.push_str(&block);
         current_chars += block_chars;
         current_words += block_words;
@@ -118,7 +136,7 @@ fn chunk_blocks(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
     if !current.is_empty() {
         out.push(current);
     }
-    out
+    Ok(out)
 }
 
 /// Return the trailing `overlap` characters of `s`, aligned to a
@@ -167,13 +185,6 @@ fn top_level_blocks(events: &[Event<'_>]) -> Vec<(usize, usize)> {
     blocks
 }
 
-fn render_markdown(events: &[Event<'_>]) -> String {
-    let mut buf = String::new();
-    pulldown_cmark_to_cmark::cmark(events.iter().cloned(), &mut buf)
-        .expect("markdown serialization failed");
-    buf
-}
-
 // Serialize the full filtered Markdown into one string, then walk
 // word boundaries. Each window is a byte-aligned slice of the
 // serialized output ending at a word boundary that fits the budget.
@@ -181,10 +192,14 @@ fn render_markdown(events: &[Event<'_>]) -> String {
 // `overlap` chars (to the next word boundary) before taking the
 // next window.
 
-fn chunk_words(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
-    let rendered = render_markdown(events);
+fn chunk_words(
+    events: &[Event<'_>],
+    params: &WindowParams,
+    md_opts: pulldown_cmark::Options,
+) -> Result<Vec<String>, SerializeError> {
+    let rendered = markdown::render(events, md_opts)?;
     if rendered.is_empty() {
-        return Vec::new();
+        return Ok(Vec::new());
     }
 
     let mut out: Vec<String> = Vec::new();
@@ -250,7 +265,7 @@ fn chunk_words(events: &[Event<'_>], params: &WindowParams) -> Vec<String> {
         };
     }
 
-    out
+    Ok(out)
 }
 
 /// Step the cursor back by roughly `overlap` characters, then land on

data/ext/inkmark/src/document.rs

@@ -7,6 +7,7 @@ use crate::heading;
 use crate::highlight;
 use crate::image;
 use crate::link;
+use crate::markdown::{self, SerializeError};
 use crate::options::{build_options, Flags};
 use crate::plain_text;
 use crate::scheme_filter::SchemeFilter;
@@ -36,13 +37,14 @@ pub fn native_to_markdown(
     source: String,
     opts_hash: Option<RHash>,
 ) -> Result<String, Error> {
-    match opts_hash {
-        None => Ok(markdown_defaults(&source)),
+    let rendered = match opts_hash {
+        None => markdown_defaults(&source),
         Some(hash) => {
             let (cm_opts, flags) = build_options(ruby, hash)?;
-            Ok(render_to_markdown(&source, cm_opts, flags))
+            render_to_markdown(&source, cm_opts, flags)
         }
-    }
+    };
+    rendered.map_err(|e| markdown::to_ruby_error(ruby, e))
 }
 
 pub fn native_to_plain_text(
@@ -71,12 +73,13 @@ fn render_defaults(source: &str) -> String {
 }
 
 /// Same as render_defaults but serializes to Markdown instead of HTML.
-fn markdown_defaults(source: &str) -> String {
+fn markdown_defaults(source: &str) -> Result<String, SerializeError> {
     let mut buf = String::with_capacity(source.len());
-    let parser = Parser::new_ext(source, default_cm_opts());
+    let cm_opts = default_cm_opts();
+    let parser = Parser::new_ext(source, cm_opts);
     let filtered = parser.map(suppress_raw_html);
-    cmark_write(filtered, &mut buf);
-    buf
+    markdown::write(filtered, &mut buf, cm_opts)?;
+    Ok(buf)
 }
 
 /// Defaults-only plain-text fast path. Mirrors `markdown_defaults`:
@@ -255,22 +258,25 @@ fn render(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> Strin
     buf
 }
 
-fn render_to_markdown(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> String {
-    let mut buf = String::with_capacity(source.len());
-
+fn render_to_markdown(
+    source: &str,
+    cm_opts: pulldown_cmark::Options,
+    flags: Flags,
+) -> Result<String, SerializeError> {
     // `content_events` strips frontmatter, so the cmark serializer never sees
     // a metadata block to re-emit—on either the streaming or buffered path.
     if !needs_buffer(&flags) {
-        cmark_write(
+        let mut buf = String::with_capacity(source.len());
+        markdown::write(
             content_events(source, cm_opts).map(stream_filter(&flags)),
             &mut buf,
-        );
-        return buf;
+            cm_opts,
+        )?;
+        return Ok(buf);
     }
 
     let events = apply_filters(content_events(source, cm_opts).collect(), &flags);
-    cmark_write(events.into_iter(), &mut buf);
-    buf
+    markdown::render(&events, markdown::serializer_options(cm_opts, &flags))
 }
 
 fn render_to_plain_text(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> String {
@@ -447,7 +453,3 @@ pub fn apply_post_handler_filters<'a>(mut events: Vec<Event<'a>>, flags: &Flags)
 
     events
 }
-
-fn cmark_write<'a, I: Iterator<Item = Event<'a>>>(events: I, buf: &mut String) {
-    pulldown_cmark_to_cmark::cmark(events, buf).expect("markdown serialization failed");
-}