inkmark 0.1.0

data/ext/inkmark/src/toc.rs

@@ -0,0 +1,221 @@
+//! Table-of-contents rendering.
+//!
+//! Converts a slice of `TocEntry` values (produced by `stats::collect`)
+//! into markdown or HTML strings.
+
+use pulldown_cmark::HeadingLevel;
+use pulldown_cmark_escape::escape_html;
+
+pub struct TocEntry {
+    pub level: HeadingLevel,
+    pub text: String,
+    pub slug: String,
+}
+
+/// Render TOC entries as a markdown list.
+///
+/// Square brackets in heading text are escaped (`[` → `\[`, `]` → `\]`) so
+/// they don't break the markdown link syntax `[text](#slug)`.
+///
+/// When `max_depth` is `Some(n)`, only headings at level ≤ n are included.
+/// `None` means no depth filtering (every heading appears).
+pub fn toc_to_markdown(entries: &[TocEntry], max_depth: Option<u8>) -> String {
+    let filtered: Vec<&TocEntry> = entries
+        .iter()
+        .filter(|e| max_depth.map_or(true, |max| level_to_u8(e.level) <= max))
+        .collect();
+    if filtered.is_empty() {
+        return String::new();
+    }
+    let min_level = filtered
+        .iter()
+        .map(|e| level_to_u8(e.level))
+        .min()
+        .unwrap_or(1);
+
+    let mut buf = String::new();
+    for entry in &filtered {
+        let indent = (level_to_u8(entry.level) - min_level) as usize * 2;
+        for _ in 0..indent {
+            buf.push(' ');
+        }
+        buf.push_str("- [");
+        // Escape [ and ] so they don't break the markdown link syntax.
+        for ch in entry.text.chars() {
+            match ch {
+                '[' => buf.push_str("\\["),
+                ']' => buf.push_str("\\]"),
+                c => buf.push(c),
+            }
+        }
+        buf.push_str("](#");
+        buf.push_str(&entry.slug);
+        buf.push_str(")\n");
+    }
+    buf
+}
+
+/// Render TOC entries as a nested HTML `<ul>` list.
+///
+/// When `max_depth` is `Some(n)`, only headings at level ≤ n are included.
+pub fn toc_to_html(entries: &[TocEntry], max_depth: Option<u8>) -> String {
+    let filtered: Vec<&TocEntry> = entries
+        .iter()
+        .filter(|e| max_depth.map_or(true, |max| level_to_u8(e.level) <= max))
+        .collect();
+    if filtered.is_empty() {
+        return String::new();
+    }
+    let min_level = filtered
+        .iter()
+        .map(|e| level_to_u8(e.level))
+        .min()
+        .unwrap_or(1);
+    let mut buf = String::new();
+    let mut open_levels: Vec<u8> = Vec::new();
+
+    for entry in &filtered {
+        let lvl = level_to_u8(entry.level);
+
+        while open_levels
+            .last()
+            .copied()
+            .unwrap_or(min_level.saturating_sub(1))
+            >= lvl
+        {
+            buf.push_str("</li>\n</ul>\n");
+            open_levels.pop();
+        }
+
+        while open_levels
+            .last()
+            .copied()
+            .unwrap_or(min_level.saturating_sub(1))
+            < lvl
+        {
+            buf.push_str("<ul>\n");
+            open_levels.push(lvl);
+        }
+
+        buf.push_str("<li><a href=\"#");
+        buf.push_str(&entry.slug);
+        buf.push_str("\">");
+
+        let _ = escape_html(&mut buf, &entry.text);
+        buf.push_str("</a>\n");
+    }
+
+    while open_levels.pop().is_some() {
+        buf.push_str("</li>\n</ul>\n");
+    }
+
+    buf
+}
+
+#[inline]
+pub fn level_to_u8(level: HeadingLevel) -> u8 {
+    match level {
+        HeadingLevel::H1 => 1,
+        HeadingLevel::H2 => 2,
+        HeadingLevel::H3 => 3,
+        HeadingLevel::H4 => 4,
+        HeadingLevel::H5 => 5,
+        HeadingLevel::H6 => 6,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use pulldown_cmark::HeadingLevel;
+
+    fn entry(level: HeadingLevel, text: &str, slug: &str) -> TocEntry {
+        TocEntry {
+            level,
+            text: text.to_string(),
+            slug: slug.to_string(),
+        }
+    }
+
+    #[test]
+    fn empty_toc_returns_empty_string() {
+        assert_eq!(toc_to_markdown(&[], None), "");
+        assert_eq!(toc_to_html(&[], None), "");
+    }
+
+    #[test]
+    fn markdown_escapes_brackets_in_heading_text() {
+        let entries = vec![entry(HeadingLevel::H1, "Arrays [1..n]", "arrays-1-n")];
+        let md = toc_to_markdown(&entries, None);
+        assert!(md.contains("\\[1..n\\]"), "brackets must be escaped: {md}");
+        assert!(md.contains("(#arrays-1-n)"));
+    }
+
+    #[test]
+    fn markdown_simple_toc() {
+        let entries = vec![
+            entry(HeadingLevel::H1, "Introduction", "introduction"),
+            entry(HeadingLevel::H2, "Background", "background"),
+        ];
+        let md = toc_to_markdown(&entries, None);
+        assert_eq!(
+            md,
+            "- [Introduction](#introduction)\n  - [Background](#background)\n"
+        );
+    }
+
+    #[test]
+    fn html_toc_escapes_text() {
+        let entries = vec![entry(HeadingLevel::H1, "A & B <C>", "a-b-c")];
+        let html = toc_to_html(&entries, None);
+        assert!(html.contains("A &amp; B &lt;C&gt;"));
+    }
+
+    #[test]
+    fn max_depth_filters_markdown() {
+        let entries = vec![
+            entry(HeadingLevel::H1, "Top", "top"),
+            entry(HeadingLevel::H2, "Mid", "mid"),
+            entry(HeadingLevel::H3, "Deep", "deep"),
+        ];
+        let md = toc_to_markdown(&entries, Some(2));
+        assert!(md.contains("[Top]"));
+        assert!(md.contains("[Mid]"));
+        assert!(!md.contains("[Deep]"));
+    }
+
+    #[test]
+    fn max_depth_filters_html() {
+        let entries = vec![
+            entry(HeadingLevel::H1, "Top", "top"),
+            entry(HeadingLevel::H2, "Mid", "mid"),
+            entry(HeadingLevel::H3, "Deep", "deep"),
+        ];
+        let html = toc_to_html(&entries, Some(2));
+        assert!(html.contains(">Top</a>"));
+        assert!(html.contains(">Mid</a>"));
+        assert!(!html.contains(">Deep</a>"));
+    }
+
+    #[test]
+    fn max_depth_one_keeps_only_h1() {
+        let entries = vec![
+            entry(HeadingLevel::H1, "Top", "top"),
+            entry(HeadingLevel::H2, "Mid", "mid"),
+        ];
+        let md = toc_to_markdown(&entries, Some(1));
+        assert_eq!(md, "- [Top](#top)\n");
+    }
+
+    #[test]
+    fn max_depth_respects_remaining_entries_for_min_level() {
+        // With h3 filtered out, min_level is h1, so h2 gets 2-space indent.
+        let entries = vec![
+            entry(HeadingLevel::H1, "Top", "top"),
+            entry(HeadingLevel::H2, "Mid", "mid"),
+            entry(HeadingLevel::H3, "Deep", "deep"),
+        ];
+        let md = toc_to_markdown(&entries, Some(2));
+        assert_eq!(md, "- [Top](#top)\n  - [Mid](#mid)\n");
+    }
+}

data/ext/inkmark/src/truncate.rs

@@ -0,0 +1,267 @@
+//! Markdown truncation for LLM / RAG pipelines.
+//!
+//! Cuts a filter-applied event stream at either a block boundary
+//! (`TruncateAt::Block`) or a Unicode word boundary
+//! (`TruncateAt::Word`), respecting optional character and word
+//! budgets. Designed as a first-stage preprocessor for embedding
+//! input, context-window budgeting, and chunk normalization.
+
+use magnus::{Error, RHash, Ruby};
+use pulldown_cmark::{Event, Parser};
+use unicode_segmentation::UnicodeSegmentation;
+
+use crate::document::apply_filters;
+use crate::options::{build_options, Flags};
+
+/// What kind of boundary to cut at.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub enum TruncateAt {
+    /// Last top-level Markdown block that fits the budget. Output is
+    /// always valid Markdown.
+    Block,
+    /// Last Unicode word boundary that fits the budget. Output is a
+    /// Markdown string but may split an open construct (code fence,
+    /// link, emphasis).
+    Word,
+}
+
+/// Parameters for a truncation pass. At least one of `chars` / `words`
+/// must be `Some`; if both are set, cut when the first of the two
+/// budgets is exhausted. `marker` ("...") counts toward the budget:
+/// if supplied, the effective content budget is reduced so that final
+/// output length stays at or under the user-given limit.
+pub struct TruncateParams {
+    pub chars: Option<usize>,
+    pub words: Option<usize>,
+    pub at: TruncateAt,
+    pub marker: Option<String>,
+}
+
+/// Full-document entry point: parse + filter + truncate.
+pub fn truncate_source(
+    source: &str,
+    cm_opts: pulldown_cmark::Options,
+    flags: &Flags,
+    params: &TruncateParams,
+) -> String {
+    let events: Vec<Event> = Parser::new_ext(source, cm_opts).collect();
+    let events = apply_filters(events, flags);
+    truncate_events(&events, params)
+}
+
+/// Ruby-facing entry point. Expects `params_hash` to contain
+/// `:chars`, `:words`, `:at` (`:block` | `:word`), and `:marker`
+/// (a String or nil). Argument validation lives on the Ruby side;
+/// we just read the values defensively here.
+pub fn native_truncate_markdown(
+    ruby: &Ruby,
+    source: String,
+    params_hash: RHash,
+    opts_hash: RHash,
+) -> Result<String, Error> {
+    let (cm_opts, flags) = build_options(ruby, opts_hash)?;
+    let params = parse_params(ruby, params_hash)?;
+    Ok(truncate_source(&source, cm_opts, &flags, &params))
+}
+
+pub fn parse_params(ruby: &Ruby, hash: RHash) -> Result<TruncateParams, Error> {
+    let chars: Option<usize> = hash.lookup(ruby.to_symbol("chars"))?;
+    let words: Option<usize> = hash.lookup(ruby.to_symbol("words"))?;
+    let at_sym: Option<String> = hash.lookup(ruby.to_symbol("at"))?;
+    let marker: Option<String> = hash.lookup(ruby.to_symbol("marker"))?;
+
+    let at = match at_sym.as_deref() {
+        Some("block") => TruncateAt::Block,
+        Some("word") => TruncateAt::Word,
+        _ => TruncateAt::Block,
+    };
+
+    Ok(TruncateParams {
+        chars,
+        words,
+        at,
+        marker,
+    })
+}
+
+/// Core truncation over a filter-applied event slice.
+pub fn truncate_events(events: &[Event<'_>], params: &TruncateParams) -> String {
+    match params.at {
+        TruncateAt::Block => truncate_at_block(events, params),
+        TruncateAt::Word => truncate_at_word(events, params),
+    }
+}
+
+fn truncate_at_block(events: &[Event<'_>], params: &TruncateParams) -> String {
+    let marker_chars = marker_chars(&params.marker);
+    let marker_words = marker_words(&params.marker);
+    let char_budget = params.chars.map(|n| n.saturating_sub(marker_chars));
+    let word_budget = params.words.map(|n| n.saturating_sub(marker_words));
+
+    let mut kept = String::new();
+    let mut used_chars: usize = 0;
+    let mut used_words: usize = 0;
+    let mut any_dropped = false;
+
+    for (start, end) in top_level_blocks(events) {
+        let block = render_markdown(&events[start..=end]);
+        let block_chars = block.trim_end().chars().count();
+        let block_words = block.unicode_words().count();
+
+        let would_exceed_chars = char_budget
+            .map(|b| used_chars + block_chars > b)
+            .unwrap_or(false);
+        let would_exceed_words = word_budget
+            .map(|b| used_words + block_words > b)
+            .unwrap_or(false);
+
+        if would_exceed_chars || would_exceed_words {
+            any_dropped = true;
+            break;
+        }
+
+        kept.push_str(&block);
+        used_chars += block_chars;
+        used_words += block_words;
+    }
+
+    // If nothing was dropped and source fit, return unchanged (no marker).
+    if !any_dropped {
+        return kept;
+    }
+
+    // First block alone exceeded the budget; honest empty return
+    if kept.is_empty() {
+        return String::new();
+    }
+
+    // Trim trailing whitespace before appending the marker so the
+    // ellipsis attaches cleanly to the last block.
+    while kept.ends_with(|c: char| c.is_whitespace()) {
+        kept.pop();
+    }
+    if let Some(marker) = &params.marker {
+        kept.push_str("\n\n");
+        kept.push_str(marker);
+    }
+    kept.push('\n');
+    kept
+}
+
+/// Return (start, end) event-index pairs for top-level blocks.
+/// A block starts at an `Event::Start` with depth 0 and ends at the
+/// matching `Event::End`. Leaf events at depth 0 (Rule, HardBreak if
+/// it ever happens, raw Html block) count as single-event blocks.
+fn top_level_blocks(events: &[Event<'_>]) -> Vec<(usize, usize)> {
+    let mut blocks = Vec::new();
+    let mut depth: i32 = 0;
+    let mut current_start: Option<usize> = None;
+
+    for (i, event) in events.iter().enumerate() {
+        match event {
+            Event::Start(_) => {
+                if depth == 0 {
+                    current_start = Some(i);
+                }
+                depth += 1;
+            }
+            Event::End(_) => {
+                depth -= 1;
+                if depth == 0 {
+                    if let Some(start) = current_start.take() {
+                        blocks.push((start, i));
+                    }
+                }
+            }
+            _ => {
+                if depth == 0 {
+                    // Standalone top-level event (Event::Rule, etc.).
+                    blocks.push((i, i));
+                }
+            }
+        }
+    }
+
+    blocks
+}
+
+fn render_markdown(events: &[Event<'_>]) -> String {
+    let mut buf = String::new();
+    pulldown_cmark_to_cmark::cmark(events.iter().cloned(), &mut buf)
+        .expect("markdown serialization failed");
+    buf
+}
+
+fn truncate_at_word(events: &[Event<'_>], params: &TruncateParams) -> String {
+    let rendered = render_markdown(events);
+    let marker_chars = marker_chars(&params.marker);
+    let marker_words = marker_words(&params.marker);
+
+    let total_chars = rendered.chars().count();
+    let total_words = rendered.unicode_words().count();
+
+    let chars_ok = params
+        .chars
+        .map(|limit| total_chars + marker_chars <= limit)
+        .unwrap_or(true);
+    let words_ok = params
+        .words
+        .map(|limit| total_words + marker_words <= limit)
+        .unwrap_or(true);
+
+    // Fits under both budgets: return unchanged, no marker.
+    if chars_ok && words_ok {
+        return rendered;
+    }
+
+    let char_budget = params.chars.map(|n| n.saturating_sub(marker_chars));
+    let word_budget = params.words.map(|n| n.saturating_sub(marker_words));
+
+    // Walk word boundaries, tracking cumulative char and word counts.
+    // `last_good_end` is the byte offset of the end of the last word
+    // segment that stays within both budgets.
+    let mut last_good_end: usize = 0;
+    let mut used_chars: usize = 0;
+    let mut used_words: usize = 0;
+
+    for (offset, segment) in rendered.split_word_bound_indices() {
+        let seg_chars = segment.chars().count();
+        let seg_is_word = segment.unicode_words().next().is_some();
+        let next_words = if seg_is_word {
+            used_words + 1
+        } else {
+            used_words
+        };
+        let next_chars = used_chars + seg_chars;
+
+        let over_chars = char_budget.map(|b| next_chars > b).unwrap_or(false);
+        let over_words = word_budget.map(|b| next_words > b).unwrap_or(false);
+        if over_chars || over_words {
+            break;
+        }
+
+        used_chars = next_chars;
+        used_words = next_words;
+        last_good_end = offset + segment.len();
+    }
+
+    let mut out = rendered[..last_good_end].to_string();
+    while out.ends_with(|c: char| c.is_whitespace()) {
+        out.pop();
+    }
+    if let Some(marker) = &params.marker {
+        out.push_str(marker);
+    }
+    out
+}
+
+fn marker_chars(marker: &Option<String>) -> usize {
+    marker.as_ref().map(|m| m.chars().count()).unwrap_or(0)
+}
+
+fn marker_words(marker: &Option<String>) -> usize {
+    marker
+        .as_ref()
+        .map(|m| m.unicode_words().count())
+        .unwrap_or(0)
+}

data/ext/inkmark/src/url_match.rs

@@ -0,0 +1,178 @@
+//! URL-allowlist matching shared between the link and image filters.
+//!
+//! Two entry points, both using `url::Url::parse` as the single source
+//! of truth for URL decomposition:
+//!
+//! - [`is_host_allowed`]: match the URL's host against a glob allowlist.
+//! - [`is_scheme_allowed`]: match the URL's scheme against a string set.
+//!
+//! Both fail-open on URLs the parser can't resolve (relative paths,
+//! anchors, protocol-relative, malformed input): such URLs have nothing
+//! to check against, so they pass through unchanged.
+
+use globset::GlobSet;
+
+/// Return true when the URL should be kept by a host allowlist.
+///
+/// - **Has a host** (http/https URLs): lowercase and match against `set`.
+/// - **No host** (relative `/foo`, anchor `#x`, `mailto:`, `tel:`,
+///   `javascript:`, malformed input): out of scope for host allowlisting,
+///   return true so the caller leaves it alone.
+///
+/// Fail-open on parse failure is deliberate: relative URLs must pass
+/// through unchanged, and "can't parse" is how `url` signals that.
+pub fn is_host_allowed(url: &str, set: &GlobSet) -> bool {
+    match url::Url::parse(url)
+        .ok()
+        .and_then(|u| u.host_str().map(str::to_ascii_lowercase))
+    {
+        Some(host) => set.is_match(&host),
+        None => true,
+    }
+}
+
+/// Return true when the URL's scheme is in the allowlist.
+///
+/// - **Parses as absolute URL**: check the scheme (lowercased by `url::Url`)
+///   against `allowed`.
+/// - **Doesn't parse** (relative `/foo`, anchor `#x`, protocol-relative
+///   `//host/x`, malformed input): no scheme to check, return true.
+///
+/// `allowed` is a `&[String]` rather than a `HashSet` because every
+/// realistic scheme allowlist has 2–5 entries, a linear scan of short
+/// strings beats any hash table on CPU cache alone at that size. The
+/// caller must pre-lowercase entries; `url::Url` normalizes schemes to
+/// lowercase at parse time, so comparing against lowercase entries is
+/// correct.
+pub fn is_scheme_allowed(url: &str, allowed: &[String]) -> bool {
+    match url::Url::parse(url) {
+        Ok(parsed) => {
+            let scheme = parsed.scheme();
+            allowed.iter().any(|s| s == scheme)
+        }
+        Err(_) => true,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{is_host_allowed, is_scheme_allowed};
+    use globset::{Glob, GlobSetBuilder};
+
+    fn host_set(patterns: &[&str]) -> globset::GlobSet {
+        let mut b = GlobSetBuilder::new();
+        for p in patterns {
+            b.add(Glob::new(p).unwrap());
+        }
+        b.build().unwrap()
+    }
+
+    fn scheme_set(schemes: &[&str]) -> Vec<String> {
+        schemes.iter().map(|s| s.to_ascii_lowercase()).collect()
+    }
+
+    #[test]
+    fn exact_host_matches() {
+        let s = host_set(&["example.net"]);
+        assert!(is_host_allowed("https://example.net/path", &s));
+        assert!(!is_host_allowed("https://evil.com/path", &s));
+    }
+
+    #[test]
+    fn subdomain_wildcard() {
+        let s = host_set(&["*.example.net"]);
+        assert!(is_host_allowed("https://cdn.example.net/a.png", &s));
+        assert!(is_host_allowed("https://deeply.nested.example.net/x", &s));
+        assert!(!is_host_allowed("https://example.net/x", &s));
+        assert!(!is_host_allowed("https://evil.com/x", &s));
+    }
+
+    #[test]
+    fn brace_alternation() {
+        let s = host_set(&["{cdn,static}.example.net"]);
+        assert!(is_host_allowed("https://cdn.example.net/x", &s));
+        assert!(is_host_allowed("https://static.example.net/x", &s));
+        assert!(!is_host_allowed("https://media.example.net/x", &s));
+    }
+
+    #[test]
+    fn case_insensitive_host() {
+        let s = host_set(&["example.net"]);
+        assert!(is_host_allowed("https://EXAMPLE.NET/path", &s));
+        assert!(is_host_allowed("HTTPS://Example.Net/path", &s));
+    }
+
+    #[test]
+    fn port_is_ignored() {
+        let s = host_set(&["example.net"]);
+        assert!(is_host_allowed("https://example.net:8443/x", &s));
+    }
+
+    #[test]
+    fn no_host_passes_through() {
+        let s = host_set(&["example.net"]);
+        assert!(is_host_allowed("/local/path", &s));
+        assert!(is_host_allowed("relative.html", &s));
+        assert!(is_host_allowed("#anchor", &s));
+        assert!(is_host_allowed("mailto:user@example.net", &s));
+        assert!(is_host_allowed("tel:+1234567890", &s));
+        assert!(is_host_allowed("javascript:alert(1)", &s));
+        assert!(is_host_allowed("", &s));
+    }
+
+    #[test]
+    fn empty_host_allowlist_blocks_all_external() {
+        let s = host_set(&[]);
+        assert!(!is_host_allowed("https://example.net/x", &s));
+        assert!(!is_host_allowed("https://anything.com/x", &s));
+        assert!(is_host_allowed("/local", &s));
+    }
+
+    #[test]
+    fn scheme_matches_allowed() {
+        let s = scheme_set(&["http", "https", "mailto"]);
+        assert!(is_scheme_allowed("https://example.net/x", &s));
+        assert!(is_scheme_allowed("http://example.net/x", &s));
+        assert!(is_scheme_allowed("mailto:user@example.net", &s));
+    }
+
+    #[test]
+    fn scheme_rejects_disallowed() {
+        let s = scheme_set(&["http", "https", "mailto"]);
+        assert!(!is_scheme_allowed("javascript:alert(1)", &s));
+        assert!(!is_scheme_allowed("vbscript:msgbox", &s));
+        assert!(!is_scheme_allowed("data:text/html,<script>", &s));
+        assert!(!is_scheme_allowed("file:///etc/passwd", &s));
+        assert!(!is_scheme_allowed("tel:+1234567890", &s));
+    }
+
+    #[test]
+    fn scheme_is_case_insensitive_via_url_crate() {
+        // url::Url lowercases the scheme at parse time, so mixed-case
+        // input matches lowercase allowlist entries.
+        let s = scheme_set(&["https"]);
+        assert!(is_scheme_allowed("HTTPS://example.net", &s));
+        assert!(is_scheme_allowed("HttpS://example.net", &s));
+    }
+
+    #[test]
+    fn unparseable_url_passes_scheme_check() {
+        // Relative, anchor-only, protocol-relative, and empty URLs can't
+        // be parsed as absolute—no scheme to check, so they pass.
+        let s = scheme_set(&["https"]);
+        assert!(is_scheme_allowed("/local/path", &s));
+        assert!(is_scheme_allowed("relative.html", &s));
+        assert!(is_scheme_allowed("#anchor", &s));
+        assert!(is_scheme_allowed("//cdn.example.net/x", &s));
+        assert!(is_scheme_allowed("", &s));
+    }
+
+    #[test]
+    fn empty_scheme_allowlist_blocks_all_absolute() {
+        let s = scheme_set(&[]);
+        assert!(!is_scheme_allowed("https://example.net", &s));
+        assert!(!is_scheme_allowed("mailto:user@example.net", &s));
+        // Relative URLs still pass—nothing to match.
+        assert!(is_scheme_allowed("/local", &s));
+    }
+}