inkmark 0.1.3 → 0.2.0

data/README.md

@@ -38,6 +38,7 @@ A very fast, feature-packed, AI-first Markdown gem for Ruby.
 - [Plain-text extraction](#plain-text-extraction)
 - [Markdown-to-Markdown pipeline](#markdown-to-markdown-pipeline)
 - [Event handlers](#event-handlers)
+- [Ractors](#ractors)
 - [Benchmarks](#benchmarks)
 - [Contributing](#contributing)
 - [Acknowledgements](#acknowledgements)
@@ -177,9 +178,15 @@ g.options.math = true
 g.options.tables = false
 g.to_html
 
-# Process-level defaults, to set in your application initializer
-Inkmark.default_options.math = true
-Inkmark.new(md).to_html  # picks up the default
+# Process-wide defaults, set once in your application initializer
+Inkmark.configure do |options|
+  options.math = true
+  options.links = { nofollow: true }
+end
+Inkmark.new(md).to_html  # picks up the defaults
+
+# Or replace them wholesale
+Inkmark.default_options = { preset: :recommended, math: true }
 ```
 
 Unknown option keys raise `ArgumentError` immediately, including via the
@@ -1029,6 +1036,21 @@ Post-render filters (`syntax_highlight`, allowlists, `images: { lazy: true }`,
   highlighter
 - Handler-set `dest=` values pass through host and scheme allowlists
 
+## Ractors
+
+Inkmark is Ractor-safe: every public API works inside non-main Ractors,
+so documents can be rendered in parallel without leaving the native fast
+path.
+
+```ruby
+workers = sources.each_slice(250).map do |slice|
+  Ractor.new(slice) do |batch|
+    batch.map { |src| Inkmark.to_html(src, options: { preset: :recommended }) }
+  end
+end
+html = workers.flat_map(&:value)  # Ruby 4.0; use #take on 3.3 and 3.4
+```
+
 ## Benchmarks
 
 Inkmark ships a benchmark harness comparing it against `kramdown`,

data/ext/inkmark/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "inkmark"
-version = "0.1.3"
+version = "0.2.0"
 edition = "2021"
 authors = ["Yaroslav Markin <yaroslav@markin.net>"]
 license = "MIT"
@@ -11,11 +11,11 @@ crate-type = ["cdylib"]
 
 [dependencies]
 magnus = { version = "0.8.2" }
-rb-sys = { version = "0.9.126", features = ["stable-api-compiled-fallback"] }
+rb-sys = { version = "0.9.130", features = ["stable-api-compiled-fallback"] }
 pulldown-cmark = { version = "0.13", default-features = false, features = ["html", "simd"] }
 pulldown-cmark-escape = "0.11"
 deunicode = "1.6"
-emojis = "0.8"
+emojis = "0.9"
 whatlang = "0.18"
 unicode-segmentation = "1.12"
 syntect = { version = "5.2", default-features = false, features = ["parsing", "html", "default-themes", "default-syntaxes", "regex-fancy"] }
@@ -28,4 +28,4 @@ pulldown-cmark-to-cmark = "22"
 rb-sys-env = "0.2.2"
 
 [dev-dependencies]
-rb-sys-test-helpers = { version = "0.2.2" }
+rb-sys-test-helpers = { version = "0.3.0" }

data/ext/inkmark/src/chunks_by_heading.rs

@@ -19,7 +19,7 @@ use magnus::{Error, RArray, RHash, Ruby};
 use pulldown_cmark::{Event, HeadingLevel, Parser, Tag, TagEnd};
 use unicode_segmentation::UnicodeSegmentation;
 
-use crate::document::apply_filters;
+use crate::document::{apply_filters, content_events};
 use crate::heading::{self, SlugDeduplicator};
 use crate::options::build_options;
 use crate::toc;
@@ -43,7 +43,8 @@ pub fn native_chunks_by_heading(
     let (cm_opts, flags) = build_options(ruby, opts_hash)?;
 
     // Parse + run the full filter pipeline, same as `to_markdown`.
-    let events: Vec<Event> = Parser::new_ext(&source, cm_opts).collect();
+    // `content_events` drops frontmatter so it never becomes a section.
+    let events: Vec<Event> = content_events(&source, cm_opts).collect();
     let events = apply_filters(events, &flags);
 
     let boundaries = find_heading_boundaries(&events);

data/ext/inkmark/src/chunks_by_size.rs

@@ -18,10 +18,10 @@
 //! adjacent chunks share context.
 
 use magnus::{Error, RArray, RHash, Ruby};
-use pulldown_cmark::{Event, Parser};
+use pulldown_cmark::Event;
 use unicode_segmentation::UnicodeSegmentation;
 
-use crate::document::apply_filters;
+use crate::document::{apply_filters, content_events};
 use crate::options::build_options;
 
 #[derive(Clone, Copy, PartialEq, Eq)]
@@ -45,7 +45,7 @@ pub fn native_chunks_by_size(
     let params = parse_params(ruby, &opts_hash)?;
     let (cm_opts, flags) = build_options(ruby, opts_hash)?;
 
-    let events: Vec<Event> = Parser::new_ext(&source, cm_opts).collect();
+    let events: Vec<Event> = content_events(&source, cm_opts).collect();
     let events = apply_filters(events, &flags);
 
     let windows = match params.at {

data/ext/inkmark/src/document.rs

@@ -1,5 +1,5 @@
 use magnus::{Error, RHash, Ruby};
-use pulldown_cmark::{html, Event, Options, Parser};
+use pulldown_cmark::{html, Event, Options, Parser, Tag, TagEnd};
 
 use crate::autolink;
 use crate::emoji;
@@ -115,6 +115,44 @@ fn hard_wrap(event: Event) -> Event {
     }
 }
 
+/// Parse `source` into the content event stream that renderers and chunkers
+/// consume.
+///
+/// YAML frontmatter is removed at this boundary: it is document *metadata*
+/// (surfaced via {Inkmark#frontmatter}), never content. pulldown-cmark's HTML
+/// renderer ignores metadata blocks and our plain-text writer discards them,
+/// but the Markdown serializer (`pulldown-cmark-to-cmark`) faithfully
+/// re-emits them as `---\n...\n---`. Rather than re-stripping after the fact
+/// in every Markdown/chunk path, we never hand those consumers the events in
+/// the first place—so `to_markdown`, `chunks_by_heading`, and
+/// `chunks_by_size` are frontmatter-free by construction, with no separate
+/// pass and no special-casing of the streaming fast path.
+///
+/// Frontmatter extraction walks the *raw* parser (see `stats::collect`), so
+/// dropping the events here does not affect the `frontmatter` accessor.
+pub fn content_events(source: &str, cm_opts: Options) -> impl Iterator<Item = Event<'_>> {
+    drop_metadata(Parser::new_ext(source, cm_opts))
+}
+
+/// Iterator adapter that filters out `Start(MetadataBlock) … End(MetadataBlock)`
+/// runs, including the raw YAML `Text` between the markers. Stateful but
+/// composes with both the streaming fast path and the buffered `.collect()`
+/// path, so a single definition serves every content consumer.
+fn drop_metadata<'a>(events: impl Iterator<Item = Event<'a>>) -> impl Iterator<Item = Event<'a>> {
+    let mut in_metadata = false;
+    events.filter(move |event| match event {
+        Event::Start(Tag::MetadataBlock(_)) => {
+            in_metadata = true;
+            false
+        }
+        Event::End(TagEnd::MetadataBlock(_)) => {
+            in_metadata = false;
+            false
+        }
+        _ => !in_metadata,
+    })
+}
+
 /// Full render: parse once, collect stats + TOC from original events,
 /// apply filters, render HTML. Returns a Ruby Hash:
 ///
@@ -219,14 +257,18 @@ fn render(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> Strin
 
 fn render_to_markdown(source: &str, cm_opts: pulldown_cmark::Options, flags: Flags) -> String {
     let mut buf = String::with_capacity(source.len());
-    let parser = Parser::new_ext(source, cm_opts);
 
+    // `content_events` strips frontmatter, so the cmark serializer never sees
+    // a metadata block to re-emit—on either the streaming or buffered path.
     if !needs_buffer(&flags) {
-        cmark_write(parser.map(stream_filter(&flags)), &mut buf);
+        cmark_write(
+            content_events(source, cm_opts).map(stream_filter(&flags)),
+            &mut buf,
+        );
         return buf;
     }
 
-    let events = apply_filters(parser.collect(), &flags);
+    let events = apply_filters(content_events(source, cm_opts).collect(), &flags);
     cmark_write(events.into_iter(), &mut buf);
     buf
 }

data/ext/inkmark/src/lib.rs

@@ -1,4 +1,6 @@
-#![forbid(unsafe_code)]
+// `deny` rather than `forbid`: the single `unsafe` block in `init` is
+// allowed explicitly; everything else stays unsafe-free.
+#![deny(unsafe_code)]
 
 use magnus::{function, prelude::*, Error, Ruby};
 
@@ -22,11 +24,34 @@ mod truncate;
 mod url_match;
 
 #[magnus::init]
+#[allow(unsafe_code)]
 fn init(ruby: &Ruby) -> Result<(), Error> {
+    // Declare every method defined below Ractor-safe. Ruby marks the methods an
+    // extension defines while `Init_*` runs as unsafe unless this flag is set,
+    // and calling such a method from a non-main Ractor raises
+    // `Ractor::UnsafeError`. The extension keeps no Ruby `VALUE`s in
+    // process-global state: the `OnceLock` caches in `highlight` hold plain
+    // syntect data (`Sync`, enforced by the compiler) and the `sym_id!`
+    // statics in `options` hold interned symbol ids, both safe to share across
+    // Ractors. This must run before `define_class` so the flag covers every
+    // method registered below.
+    //
+    // SAFETY: `rb_ext_ractor_safe` takes no pointers and only flips a flag on
+    // the VM's extension-loading state. Its one precondition is being called
+    // while `Init_*` runs, which is exactly where `#[magnus::init]` invokes
+    // this function.
+    unsafe { rb_sys::rb_ext_ractor_safe(true) };
+
     let inkmark = ruby.define_class("Inkmark", ruby.class_object())?;
     inkmark.define_singleton_method("_native_to_html", function!(document::native_to_html, 2))?;
-    inkmark.define_singleton_method("_native_to_markdown", function!(document::native_to_markdown, 2))?;
-    inkmark.define_singleton_method("_native_to_plain_text", function!(document::native_to_plain_text, 2))?;
+    inkmark.define_singleton_method(
+        "_native_to_markdown",
+        function!(document::native_to_markdown, 2),
+    )?;
+    inkmark.define_singleton_method(
+        "_native_to_plain_text",
+        function!(document::native_to_plain_text, 2),
+    )?;
     inkmark.define_singleton_method(
         "_native_chunks_by_heading",
         function!(chunks_by_heading::native_chunks_by_heading, 2),

data/ext/inkmark/src/truncate.rs

@@ -7,10 +7,10 @@
 //! input, context-window budgeting, and chunk normalization.
 
 use magnus::{Error, RHash, Ruby};
-use pulldown_cmark::{Event, Parser};
+use pulldown_cmark::Event;
 use unicode_segmentation::UnicodeSegmentation;
 
-use crate::document::apply_filters;
+use crate::document::{apply_filters, content_events};
 use crate::options::{build_options, Flags};
 
 /// What kind of boundary to cut at.
@@ -44,7 +44,7 @@ pub fn truncate_source(
     flags: &Flags,
     params: &TruncateParams,
 ) -> String {
-    let events: Vec<Event> = Parser::new_ext(source, cm_opts).collect();
+    let events: Vec<Event> = content_events(source, cm_opts).collect();
     let events = apply_filters(events, flags);
     truncate_events(&events, params)
 }

data/lib/inkmark/options.rb

@@ -22,22 +22,28 @@ class Inkmark
     # Per-element-policy schemas. Each entry is +{ default:, types: }+; the
     # validators use +types+ for type checking and +default+ to seed fresh
     # nested hashes. Keep in sync with {NESTED_TO_FLAT}.
+    #
+    # Every constant in this class is frozen all the way down, not just
+    # at the top level: a non-main Ractor may only read constants whose
+    # whole object graph is frozen (+Ractor.shareable?+), so the inner
+    # Hashes and Arrays are frozen explicitly and the larger nested
+    # tables go through +Ractor.make_shareable+.
     HEADINGS_SCHEMA = {
-      attributes: {default: false, types: [TrueClass, FalseClass]},
-      ids: {default: false, types: [TrueClass, FalseClass]}
+      attributes: {default: false, types: [TrueClass, FalseClass].freeze}.freeze,
+      ids: {default: false, types: [TrueClass, FalseClass].freeze}.freeze
     }.freeze
 
     IMAGES_SCHEMA = {
-      lazy: {default: false, types: [TrueClass, FalseClass]},
-      allowed_hosts: {default: nil, types: [NilClass, Array]},
-      allowed_schemes: {default: nil, types: [NilClass, Array]}
+      lazy: {default: false, types: [TrueClass, FalseClass].freeze}.freeze,
+      allowed_hosts: {default: nil, types: [NilClass, Array].freeze}.freeze,
+      allowed_schemes: {default: nil, types: [NilClass, Array].freeze}.freeze
     }.freeze
 
     LINKS_SCHEMA = {
-      autolink: {default: false, types: [TrueClass, FalseClass]},
-      nofollow: {default: false, types: [TrueClass, FalseClass]},
-      allowed_hosts: {default: nil, types: [NilClass, Array]},
-      allowed_schemes: {default: nil, types: [NilClass, Array]}
+      autolink: {default: false, types: [TrueClass, FalseClass].freeze}.freeze,
+      nofollow: {default: false, types: [TrueClass, FalseClass].freeze}.freeze,
+      allowed_hosts: {default: nil, types: [NilClass, Array].freeze}.freeze,
+      allowed_schemes: {default: nil, types: [NilClass, Array].freeze}.freeze
     }.freeze
 
     # Registry of nested hash options => their schemas. Iterated by the
@@ -52,7 +58,7 @@ class Inkmark
     # Map from +(parent, child)+ user-facing keys to the flat key name the
     # Rust side reads. Used by {#to_native_hash} / {#to_native_hash_frozen}
     # to serialize the user-shaped hash into the FFI wire format.
-    NESTED_TO_FLAT = {
+    NESTED_TO_FLAT = Ractor.make_shareable({
       [:headings, :attributes] => :heading_attributes,
       [:headings, :ids] => :heading_ids,
       [:images, :lazy] => :lazy_images,
@@ -62,7 +68,7 @@ class Inkmark
       [:links, :nofollow] => :nofollow_external_links,
       [:links, :allowed_hosts] => :allowed_link_hosts,
       [:links, :allowed_schemes] => :allowed_link_schemes
-    }.freeze
+    })
 
     # Build a frozen defaults hash for a nested schema from its +default+
     # entries.
@@ -200,11 +206,11 @@ class Inkmark
     # the accepted type set (nil-default-but-Array-when-set, polymorphic
     # +toc+, nested-hash element-policy groups).
     TYPES = {
-      extract: [NilClass, Hash],
-      toc: [TrueClass, FalseClass, Hash],
-      headings: [Hash],
-      images: [Hash],
-      links: [Hash]
+      extract: [NilClass, Hash].freeze,
+      toc: [TrueClass, FalseClass, Hash].freeze,
+      headings: [Hash].freeze,
+      images: [Hash].freeze,
+      links: [Hash].freeze
     }.freeze
 
     # Element kinds accepted inside +extract: { ... }+. Mirrors the match
@@ -237,7 +243,7 @@ class Inkmark
     #   The GFM tagfilter stays on. **Dangerous.** Use only for content
     #   the caller fully trusts (internal team-authored docs). The
     #   caller is fully responsible for sanitizing output.
-    PRESETS = {
+    PRESETS = Ractor.make_shareable({
       commonmark: {
         gfm: false,
         gfm_tag_filter: false,
@@ -245,7 +251,7 @@ class Inkmark
         strikethrough: false,
         tasklists: false,
         footnotes: false
-      }.freeze,
+      },
 
       gfm: {
         gfm: true,
@@ -254,7 +260,7 @@ class Inkmark
         strikethrough: true,
         tasklists: true,
         footnotes: true
-      }.freeze,
+      },
 
       recommended: {
         gfm: true,
@@ -272,7 +278,7 @@ class Inkmark
         syntax_highlight: true,
         hard_wrap: true,
         frontmatter: true
-      }.freeze,
+      },
 
       trusted: {
         gfm: true,
@@ -290,8 +296,8 @@ class Inkmark
         syntax_highlight: true,
         hard_wrap: true,
         frontmatter: true
-      }.freeze
-    }.freeze
+      }
+    })
 
     # Preset applied by {#initialize} when the caller doesn't pass
     # +preset:+. +:gfm+ matches {DEFAULTS}, so the default constructor
@@ -387,7 +393,16 @@ class Inkmark
     #   hashes; the input value as-is otherwise)
     # @raise [ArgumentError] if +key+ is unknown, or the value (or any
     #   nested sub-value) has the wrong type
+    # @raise [FrozenError] if this instance is frozen (as
+    #   {Inkmark.default_options} always is)
     def []=(key, value)
+      if frozen?
+        raise FrozenError.new(
+          "can't modify frozen #{self.class}: use Inkmark.configure to change " \
+          "the process-wide defaults, or dup for a mutable copy",
+          receiver: self
+        )
+      end
       validate_key!(key)
       # Deep-merge partial nested-hash overrides (+:headings+,
       # +:images+, +:links+) so callers pass only the sub-keys they
@@ -429,11 +444,17 @@ class Inkmark
     # FFI calls that don't need to add per-call params. The cache is
     # invalidated in {#[]=} and {#initialize_copy}.
     #
-    # @return [Hash{Symbol => Object}] frozen, shared across calls until
-    #   a mutation invalidates it
+    # The hash is frozen all the way down, as a copy: nested Arrays and
+    # Hashes are duplicated before freezing so caller-supplied values
+    # (an +allowed_hosts+ Array, say) stay mutable in the caller's hands.
+    # A deeply frozen memo is what lets a frozen +Options+ be shared
+    # across Ractors.
+    #
+    # @return [Hash{Symbol => Object}] deeply frozen, shared across calls
+    #   until a mutation invalidates it
     # @api private
     def to_native_hash_frozen
-      @frozen_native_hash ||= build_native_hash.freeze
+      @frozen_native_hash ||= deep_frozen_copy(build_native_hash)
     end
 
     # Return a new Options instance with +other+'s values applied on top.
@@ -457,6 +478,18 @@ class Inkmark
     end
     alias_method :eql?, :==
 
+    # Freeze this instance. The memoized FFI hash is computed first, while
+    # the instance is still mutable, so {#to_native_hash_frozen} never has
+    # to write to a frozen object. +Ractor.make_shareable+ calls +freeze+
+    # on every object it visits, which is what makes a shared
+    # {Inkmark.default_options} renderable from any Ractor.
+    #
+    # @return [self]
+    def freeze
+      to_native_hash_frozen
+      super
+    end
+
     # Duplicate this instance, deep-copying the internal values hash so the
     # clone is fully independent from the original.
     def initialize_copy(orig)
@@ -466,14 +499,26 @@ class Inkmark
       @frozen_native_hash = nil
     end
 
-    # @!macro [attach] inkmark_options_accessor
-    #   @!attribute [rw] $1
-    #     Reader and writer for the +$1+ option. The writer routes through
-    #     {#[]=} so key validation and (for nested groups) deep-merge apply
-    #     uniformly.
+    # Reader and writer for every option key. The writer routes through
+    # {#[]=} so key validation and (for nested groups) deep-merge apply
+    # uniformly.
+    #
+    # Generated from source strings rather than +define_method+ blocks:
+    # a block-defined method carries its Proc, and Ruby refuses to call
+    # such a method from a non-main Ractor ("defined with an un-shareable
+    # Proc in a different Ractor"). Plain +def+ bodies have no such
+    # baggage. +key+ is always a Symbol from {DEFAULTS}, so interpolating
+    # it is safe.
     DEFAULTS.each_key do |key|
-      define_method(key) { @values[key] }
-      define_method("#{key}=") { |value| self[key] = value }
+      class_eval(<<~RUBY, __FILE__, __LINE__ + 1)
+        def #{key}                # def tables
+          @values[:#{key}]        #   @values[:tables]
+        end                       # end
+
+        def #{key}=(value)        # def tables=(value)
+          self[:#{key}] = value   #   self[:tables] = value
+        end                       # end
+      RUBY
     end
 
     private
@@ -517,6 +562,19 @@ class Inkmark
     def validate_key!(key) = self.class.send(:validate_key!, key)
     def validate_value!(key, value) = self.class.send(:validate_value!, key, value)
 
+    # Return a frozen copy of +value+ with every nested Hash, Array, and
+    # String frozen too. Scalars (booleans, nil, Integers, Symbols) are
+    # returned as is. Only the container shapes {#build_native_hash} can
+    # produce are handled.
+    def deep_frozen_copy(value)
+      case value
+      when Hash then value.transform_values { |v| deep_frozen_copy(v) }.freeze
+      when Array then value.map { |v| deep_frozen_copy(v) }.freeze
+      when String then -value
+      else value
+      end
+    end
+
     # Build the Rust-facing flat hash: nested element-policy hashes expand
     # into their flat keys via {NESTED_TO_FLAT}; the internal +@toc_depth+
     # is injected when set.
@@ -624,10 +682,13 @@ class Inkmark
     # +options: { preset: :name }+ call pattern, which would otherwise
     # build a fresh +Options+ instance (seed defaults, 6–14 +[]=+
     # with validation, +build_native_hash+) on every call. The cached
-    # hashes are frozen and safe to share across threads.
-    PRESETS_NATIVE_HASH = PRESETS.keys.each_with_object({}) do |name, h|
-      h[name] = new(preset: name).to_native_hash_frozen
-    end.freeze
+    # hashes are deeply frozen and safe to share across threads and
+    # Ractors.
+    PRESETS_NATIVE_HASH = Ractor.make_shareable(
+      PRESETS.keys.each_with_object({}) do |name, h|
+        h[name] = new(preset: name).to_native_hash_frozen
+      end
+    )
 
     # Build a Rust-facing flat hash from +overrides+ without allocating
     # an +Options+ instance or walking +build_native_hash+. Starts from

data/lib/inkmark/version.rb

@@ -2,5 +2,5 @@
 
 class Inkmark
   # Current gem version.
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end