commonmarker 1.0.4 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a2600e08f80e8c7d68074711c54961feb13561eeaa0a7f5f46e01906d26405f
-  data.tar.gz: 35475ea11ba513d261e7d132029bcdb1c515f1a91b8e69b417f0d389f66ce37a
+  metadata.gz: d30b2618d4b982b55e05eaa0cf53b7cbd87fbc38a643c1ee937e21184f15d0b7
+  data.tar.gz: 0bafeaf22b6a84423e437dcd8b4d0ba3b5beadd25868f67d61e31fd70152feea
 SHA512:
-  metadata.gz: ff0cfd2e68b4900c187583a3fd878df0286ee21b1ef57467c1c8b6c68d106a8644b4827de9ae351b543b87ba373e0d03173a824817d58e700696448acab9d4ff
-  data.tar.gz: 5ccdcd58ea9865d36adf6c4ceec4b6d35e15d1f2e03897d35005180e4488f69c942b888768b1ee25296b00b138c3d3a2424912abca05e5f520f6b9587b1926ff
+  metadata.gz: 2d183a2725c73d2c9c2464d780599694566eeaa15736a90db20035977393200c4e1e67e03c1194afb83e60b27e7ddd7fa4052abd2a09015663aa3736e0cd8509
+  data.tar.gz: 15dac96c9f6797553d743829058e0c3ec3bbed065987be0b534534700b71b7c057027bb2bcaacf912b99eccfc9f191663002653c8313a82d4e7a943b35e54cea

data/Cargo.lock

@@ -221,19 +221,22 @@ version = "1.0.0"
 dependencies = [
  "comrak",
  "magnus",
+ "rctree",
  "syntect",
+ "typed-arena",
 ]
 
 [[package]]
 name = "comrak"
-version = "0.20.0"
+version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f18e72341e6cdc7489cffb76f993812a14a906db54dedb020044ccc211dcaae"
+checksum = "ab5681b26a36dc46800a908f7d932672a4e82f74759419dd4018bf40658457d5"
 dependencies = [
  "clap",
  "derive_builder",
  "emojis",
  "entities",
+ "in-place",
  "memchr",
  "once_cell",
  "regex",
@@ -376,6 +379,12 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "fastrand"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+
 [[package]]
 name = "flate2"
 version = "1.0.28"
@@ -416,6 +425,15 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
+[[package]]
+name = "in-place"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cdb69f3adfd5f493210cece799f4620417bf9965bc1536c22ae0e29ba27a8c0"
+dependencies = [
+ "tempfile",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.1.0"
@@ -446,9 +464,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.151"
+version = "0.2.154"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"
 
 [[package]]
 name = "libloading"
@@ -489,9 +507,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "magnus"
-version = "0.6.2"
+version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4778544796676e8428e9c622460ebf284bea52d8b10db3aeb449d8b5e61b3a13"
+checksum = "0fc7a31fb0b64761e3cd09a6975577601fccc5f08b8fc9245064fc4f71ed6a9d"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -676,6 +694,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
 
+[[package]]
+name = "rctree"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e03e7866abec1101869ffa8e2c8355c4c2419d0214ece0cc3e428e5b94dea6e9"
+
 [[package]]
 name = "regex"
 version = "1.10.2"
@@ -685,7 +709,7 @@ dependencies = [
  "aho-corasick",
  "memchr",
  "regex-automata",
- "regex-syntax 0.8.2",
+ "regex-syntax",
 ]
 
 [[package]]
@@ -696,15 +720,9 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.8.2",
+ "regex-syntax",
 ]
 
-[[package]]
-name = "regex-syntax"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
-
 [[package]]
 name = "regex-syntax"
 version = "0.8.2"
@@ -719,9 +737,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
 name = "rustix"
-version = "0.38.28"
+version = "0.38.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
 dependencies = [
  "bitflags 2.4.1",
  "errno",
@@ -796,9 +814,9 @@ checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
 
 [[package]]
 name = "shlex"
-version = "1.2.0"
+version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "siphasher"
@@ -846,9 +864,9 @@ dependencies = [
 
 [[package]]
 name = "syntect"
-version = "5.1.0"
+version = "5.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e02b4b303bf8d08bfeb0445cba5068a3d306b6baece1d5582171a9bf49188f91"
+checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
 dependencies = [
  "bincode",
  "bitflags 1.3.2",
@@ -858,14 +876,27 @@ dependencies = [
  "once_cell",
  "onig",
  "plist",
- "regex-syntax 0.7.5",
+ "regex-syntax",
  "serde",
+ "serde_derive",
  "serde_json",
  "thiserror",
  "walkdir",
  "yaml-rust",
 ]
 
+[[package]]
+name = "tempfile"
+version = "3.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "rustix",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "terminal_size"
 version = "0.3.0"

data/README.md

@@ -1,8 +1,5 @@
 # Commonmarker
 
-> **Note**
-> This README refers to the behavior in the new 1.0.0.pre gem.
-
 Ruby wrapper for Rust's [comrak](https://github.com/kivikakk/comrak) crate.
 
 It passes all of the CommonMark test suite, and is therefore spec-complete. It also includes extensions to the CommonMark spec as documented in the [GitHub Flavored Markdown spec](http://github.github.com/gfm/), such as support for tables, strikethroughs, and autolinking.
@@ -34,10 +31,108 @@ require 'commonmarker'
 Commonmarker.to_html('"Hi *there*"', options: {
     parse: { smart: true }
 })
-# <p>“Hi <em>there</em>”</p>\n
+# => <p>“Hi <em>there</em>”</p>\n
+```
+
+(The second argument is optional--[see below](#options-and-plugins) for more information.)
+
+### Generating a document
+
+You can also parse a string to receive a `:document` node. You can then print that node to HTML, iterate over the children, and do other fun node stuff. For example:
+
+```ruby
+require 'commonmarker'
+
+doc = Commonmarker.parse("*Hello* world", options: {
+    parse: { smart: true }
+})
+puts(doc.to_html) # => <p><em>Hello</em> world</p>\n
+
+doc.walk do |node|
+  puts node.type # => [:document, :paragraph, :emph, :text, :text]
+end
 ```
 
-The second argument is optional--[see below](#options) for more information.
+(The second argument is optional--[see below](#options-and-plugins) for more information.)
+
+When it comes to modifying the document, you can perform the following operations:
+
+- `insert_before`
+- `insert_after`
+- `prepend_child`
+- `append_child`
+- `delete`
+
+You can also get the source position of a node by calling `source_position`:
+
+```ruby
+doc = Commonmarker.parse("*Hello* world")
+puts doc.first_child.first_child.source_position
+# => {:start_line=>1, :start_column=>1, :end_line=>1, :end_column=>7}
+```
+
+You can also modify the following attributes:
+
+- `url`
+- `title`
+- `header_level`
+- `list_type`
+- `list_start`
+- `list_tight`
+- `fence_info`
+
+#### Example: Walking the AST
+
+You can use `walk` or `each` to iterate over nodes:
+
+- `walk` will iterate on a node and recursively iterate on a node's children.
+- `each` will iterate on a node and its children, but no further.
+
+```ruby
+require 'commonmarker'
+
+# parse some string
+doc = Commonmarker.parse("# The site\n\n [GitHub](https://www.github.com)")
+
+# Walk tree and print out URLs for links
+doc.walk do |node|
+  if node.type == :link
+    printf("URL = %s\n", node.url)
+  end
+end
+# => URL = https://www.github.com
+
+# Transform links to regular text
+doc.walk do |node|
+  if node.type == :link
+    node.insert_before(node.first_child)
+    node.delete
+  end
+end
+# => <h1><a href=\"#the-site\"></a>The site</h1>\n<p>GitHub</p>\n
+```
+
+#### Example: Converting a document back into raw CommonMark
+
+You can use `to_commonmark` on a node to render it as raw text:
+
+```ruby
+require 'commonmarker'
+
+# parse some string
+doc = Commonmarker.parse("# The site\n\n [GitHub](https://www.github.com)")
+
+# Transform links to regular text
+doc.walk do |node|
+  if node.type == :link
+    node.insert_before(node.first_child)
+    node.delete
+  end
+end
+
+doc.to_commonmark
+# => # The site\n\nGitHub\n
+```
 
 ## Options and plugins
 
@@ -56,21 +151,23 @@ Note that there is a distinction in comrak for "parse" options and "render" opti
 
 ### Parse options
 
-| Name                  | Description                                                                          | Default |
-| --------------------- | ------------------------------------------------------------------------------------ | ------- |
-| `smart`               | Punctuation (quotes, full-stops and hyphens) are converted into 'smart' punctuation. | `false` |
-| `default_info_string` | The default info string for fenced code blocks.                                      | `""`    |
+| Name                  | Description                                                                                                                                | Default |
+| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | ------- |
+| `smart`               | Punctuation (quotes, full-stops and hyphens) are converted into 'smart' punctuation.                                                       | `false` |
+| `default_info_string` | The default info string for fenced code blocks.                                                                                            | `""`    |
+| `relaxed_autolinks`   | Enable relaxing of the autolink extension parsing, allowing links to be recognized when in brackets, as well as permitting any url scheme. | `false` |
 
 ### Render options
 
-| Name              | Description                                                                                            | Default |
-| ----------------- | ------------------------------------------------------------------------------------------------------ | ------- |
-| `hardbreaks`      | [Soft line breaks](http://spec.commonmark.org/0.27/#soft-line-breaks) translate into hard line breaks. | `true`  |
-| `github_pre_lang` | GitHub-style `<pre lang="xyz">` is used for fenced code blocks with info tags.                         | `true`  |
-| `width`           | The wrap column when outputting CommonMark.                                                            | `80`    |
-| `unsafe`          | Allow rendering of raw HTML and potentially dangerous links.                                           | `false` |
-| `escape`          | Escape raw HTML instead of clobbering it.                                                              | `false` |
-| `sourcepos`       | Include source position attribute in HTML and XML output.                                              | `false` |
+| Name                 | Description                                                                                            | Default |
+| -------------------- | ------------------------------------------------------------------------------------------------------ | ------- |
+| `hardbreaks`         | [Soft line breaks](http://spec.commonmark.org/0.27/#soft-line-breaks) translate into hard line breaks. | `true`  |
+| `github_pre_lang`    | GitHub-style `<pre lang="xyz">` is used for fenced code blocks with info tags.                         | `true`  |
+| `width`              | The wrap column when outputting CommonMark.                                                            | `80`    |
+| `unsafe`             | Allow rendering of raw HTML and potentially dangerous links.                                           | `false` |
+| `escape`             | Escape raw HTML instead of clobbering it.                                                              | `false` |
+| `sourcepos`          | Include source position attribute in HTML and XML output.                                              | `false` |
+| `escaped_char_spans` | Wrap escaped characters in span tags                                                                   | `true`  |
 
 As well, there are several extensions which you can toggle in the same manner:
 
@@ -83,19 +180,21 @@ Commonmarker.to_html('"Hi *there*"', options: {
 
 ### Extension options
 
-| Name                     | Description                                                                                                         | Default |
-| ------------------------ | ------------------------------------------------------------------------------------------------------------------- | ------- |
-| `strikethrough`          | Enables the [strikethrough extension](https://github.github.com/gfm/#strikethrough-extension-) from the GFM spec.   | `true`  |
-| `tagfilter`              | Enables the [tagfilter extension](https://github.github.com/gfm/#disallowed-raw-html-extension-) from the GFM spec. | `true`  |
-| `table`                  | Enables the [table extension](https://github.github.com/gfm/#tables-extension-) from the GFM spec.                  | `true`  |
-| `autolink`               | Enables the [autolink extension](https://github.github.com/gfm/#autolinks-extension-) from the GFM spec.            | `true`  |
-| `tasklist`               | Enables the [task list extension](https://github.github.com/gfm/#task-list-items-extension-) from the GFM spec.     | `true`  |
-| `superscript`            | Enables the superscript Comrak extension.                                                                           | `false` |
-| `header_ids`             | Enables the header IDs Comrak extension. from the GFM spec.                                                         | `""`    |
-| `footnotes`              | Enables the footnotes extension per `cmark-gfm`.                                                                    | `false` |
-| `description_lists`      | Enables the description lists extension.                                                                            | `false` |
-| `front_matter_delimiter` | Enables the front matter extension.                                                                                 | `""`    |
-| `shortcodes`             | Enables the shortcodes extension.                                                                                   | `true`  |
+| Name                        | Description                                                                                                         | Default |
+| --------------------------- | ------------------------------------------------------------------------------------------------------------------- | ------- |
+| `strikethrough`             | Enables the [strikethrough extension](https://github.github.com/gfm/#strikethrough-extension-) from the GFM spec.   | `true`  |
+| `tagfilter`                 | Enables the [tagfilter extension](https://github.github.com/gfm/#disallowed-raw-html-extension-) from the GFM spec. | `true`  |
+| `table`                     | Enables the [table extension](https://github.github.com/gfm/#tables-extension-) from the GFM spec.                  | `true`  |
+| `autolink`                  | Enables the [autolink extension](https://github.github.com/gfm/#autolinks-extension-) from the GFM spec.            | `true`  |
+| `tasklist`                  | Enables the [task list extension](https://github.github.com/gfm/#task-list-items-extension-) from the GFM spec.     | `true`  |
+| `superscript`               | Enables the superscript Comrak extension.                                                                           | `false` |
+| `header_ids`                | Enables the header IDs Comrak extension. from the GFM spec.                                                         | `""`    |
+| `footnotes`                 | Enables the footnotes extension per `cmark-gfm`.                                                                    | `false` |
+| `description_lists`         | Enables the description lists extension.                                                                            | `false` |
+| `front_matter_delimiter`    | Enables the front matter extension.                                                                                 | `""`    |
+| `shortcodes`                | Enables the shortcodes extension.                                                                                   | `true`  |
+| `multiline_block_quotes`    | Enables the multiline block quotes extension.                                                                       | `false` |
+| `math_dollars`, `math_code` | Enables the math extension.                                                                                         | `false` |
 
 For more information on these options, see [the comrak documentation](https://github.com/kivikakk/comrak#usage).
 
@@ -205,26 +304,32 @@ If there were no errors, you're done! Otherwise, make sure to follow the comrak
 
 ## Benchmarks
 
-Some rough benchmarks:
-
 ```
-$ bundle exec rake benchmark
-
+❯ bundle exec rake benchmark
 input size = 11064832 bytes
 
+ruby 3.3.0 (2023-12-25 revision 5124f9ac75) [arm64-darwin23]
 Warming up --------------------------------------
-           redcarpet     2.000  i/100ms
-commonmarker with to_html
-                         1.000  i/100ms
-            kramdown     1.000  i/100ms
+  Markly.render_html     1.000 i/100ms
+Markly::Node#to_html     1.000 i/100ms
+Commonmarker.to_html     1.000 i/100ms
+Commonmarker::Node.to_html
+                         1.000 i/100ms
+Kramdown::Document#to_html
+                         1.000 i/100ms
 Calculating -------------------------------------
-           redcarpet     22.317  (± 4.5%) i/s -    112.000  in   5.036374s
-commonmarker with to_html
-                          5.815  (± 0.0%) i/s -     30.000  in   5.168869s
-            kramdown      0.327  (± 0.0%) i/s -      2.000  in   6.121486s
+  Markly.render_html     15.606 (±25.6%) i/s -     71.000 in   5.047132s
+Markly::Node#to_html     15.692 (±25.5%) i/s -     72.000 in   5.095810s
+Commonmarker.to_html      4.482 (± 0.0%) i/s -     23.000 in   5.137680s
+Commonmarker::Node.to_html
+                          5.092 (±19.6%) i/s -     25.000 in   5.072220s
+Kramdown::Document#to_html
+                          0.379 (± 0.0%) i/s -      2.000 in   5.277770s
 
 Comparison:
-           redcarpet:       22.3 i/s
-commonmarker with to_html:        5.8 i/s - 3.84x  (± 0.00) slower
-            kramdown:        0.3 i/s - 68.30x  (± 0.00) slower
+Markly::Node#to_html:       15.7 i/s
+  Markly.render_html:       15.6 i/s - same-ish: difference falls within error
+Commonmarker::Node.to_html:        5.1 i/s - 3.08x  slower
+Commonmarker.to_html:        4.5 i/s - 3.50x  slower
+Kramdown::Document#to_html:        0.4 i/s - 41.40x  slower
 ```

data/ext/commonmarker/Cargo.toml

@@ -7,8 +7,10 @@ publish = false
 
 [dependencies]
 magnus = "0.6"
-comrak = { version = "0.20", features = ["shortcodes"] }
-syntect = { version = "5.1", features = ["plist-load"] }
+comrak = { version = "0.23", features = ["shortcodes"] }
+syntect = { version = "5.2", features = ["plist-load"] }
+typed-arena = "2.0"
+rctree = "0.6"
 
 [lib]
 name = "commonmarker"

data/ext/commonmarker/src/lib.rs

@@ -5,13 +5,14 @@ use std::path::PathBuf;
 use ::syntect::highlighting::ThemeSet;
 use comrak::{
     adapters::SyntaxHighlighterAdapter,
-    markdown_to_html, markdown_to_html_with_plugins,
+    markdown_to_html, markdown_to_html_with_plugins, parse_document,
     plugins::syntect::{SyntectAdapter, SyntectAdapterBuilder},
     ComrakOptions, ComrakPlugins,
 };
 use magnus::{
     define_module, exception, function, r_hash::ForEach, scan_args, Error, RHash, Symbol, Value,
 };
+use node::CommonmarkerNode;
 
 mod options;
 use options::iterate_options_hash;
@@ -21,11 +22,36 @@ use plugins::{
     syntax_highlighting::{fetch_syntax_highlighter_path, fetch_syntax_highlighter_theme},
     SYNTAX_HIGHLIGHTER_PLUGIN,
 };
+use typed_arena::Arena;
 
+mod node;
 mod utils;
 
 pub const EMPTY_STR: &str = "";
 
+fn commonmark_parse(args: &[Value]) -> Result<CommonmarkerNode, magnus::Error> {
+    let args = scan_args::scan_args::<_, (), (), (), _, ()>(args)?;
+    let (rb_commonmark,): (String,) = args.required;
+
+    let kwargs =
+        scan_args::get_kwargs::<_, (), (Option<RHash>,), ()>(args.keywords, &[], &["options"])?;
+    let (rb_options,) = kwargs.optional;
+
+    let mut comrak_options = ComrakOptions::default();
+
+    if let Some(rb_options) = rb_options {
+        rb_options.foreach(|key: Symbol, value: RHash| {
+            iterate_options_hash(&mut comrak_options, key, value)?;
+            Ok(ForEach::Continue)
+        })?;
+    }
+
+    let arena = Arena::new();
+    let root = parse_document(&arena, &rb_commonmark, &comrak_options);
+
+    CommonmarkerNode::new_from_comrak_node(root)
+}
+
 fn commonmark_to_html(args: &[Value]) -> Result<String, magnus::Error> {
     let args = scan_args::scan_args::<_, (), (), (), _, ()>(args)?;
     let (rb_commonmark,): (String,) = args.required;
@@ -145,9 +171,13 @@ fn commonmark_to_html(args: &[Value]) -> Result<String, magnus::Error> {
 
 #[magnus::init]
 fn init() -> Result<(), Error> {
-    let module = define_module("Commonmarker")?;
+    let m_commonmarker = define_module("Commonmarker")?;
+
+    m_commonmarker.define_module_function("commonmark_parse", function!(commonmark_parse, -1))?;
+    m_commonmarker
+        .define_module_function("commonmark_to_html", function!(commonmark_to_html, -1))?;
 
-    module.define_module_function("commonmark_to_html", function!(commonmark_to_html, -1))?;
+    node::init(m_commonmarker).expect("cannot define Commonmarker::Node class");
 
     Ok(())
 }