markdown_it_ruby 0.1.1

data/Cargo.toml

@@ -0,0 +1,7 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/markdown_it_ruby"]
+resolver = "2"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Koji Onishi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+# MarkdownItRubyfiedSample
+
+The gem is built based on [markdown-it-rust](https://github.com/markdown-it-rust/markdown-it), which provides a super powerful parser mechanisms with highly-customizeable modular design. (I recommend you to check markdown-it-rust github page if you are interested!)
+
+On top of default parser plugins, this gem adds a few utility add-ons that can be activated via options
+Packaged as Ruby gem (NOTE: it's still experimental)
+
+## Installation
+
+Just clone the repository and try `bundle exec rake compile` to compile the executable.
+(bundler >= 2.4, RubyGems 3.4.6 required)
+
+## Usage
+
+* using with console
+```
+$ bin/console
+> 
+```
+
+* benchmark
+```
+$ cd bench
+$ ruby bench.rb
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/markdown_it_ruby. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/markdown_it_ruby/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the MarkdownItRubyfiedSample project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/markdown_it_ruby/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+require "rb_sys/extensiontask"
+
+task build: :compile
+
+GEMSPEC = Gem::Specification.load("markdown_it_ruby.gemspec")
+
+RbSys::ExtensionTask.new("markdown_it_ruby", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/markdown_it_ruby"
+end
+
+task default: %i[compile spec rubocop]

data/ext/markdown_it_ruby/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "markdown_it_ruby"
+version = "0.1.1"
+edition = "2021"
+authors = ["Koji Onishi <fursich0@gmail.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+doc = false
+
+[dependencies]
+magnus       = { version = "0.7.1" }
+markdown-it  = ">= 0.6"
+url          = ">= 2.5"
+regex        = ">= 1.10"
+uuid         = { version = ">= 1.8", features = ["v4"] }

data/ext/markdown_it_ruby/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("markdown_it_ruby/markdown_it_ruby")

data/ext/markdown_it_ruby/src/driver/options.rs

@@ -0,0 +1,298 @@
+use markdown_it::parser::extset::MarkdownItExt;
+use markdown_it::MarkdownIt;
+use std::collections::HashMap;
+use url::Url;
+
+#[derive(Debug)]
+pub struct MarkdonwItOptions {
+    options: HashMap<String, String>,
+}
+
+#[derive(Debug, Clone)]
+pub struct InternalDomain {
+    base_url: Url,
+}
+
+impl MarkdownItExt for MarkdonwItOptions {}
+impl Clone for MarkdonwItOptions {
+    fn clone(&self) -> Self {
+        Self {
+            options: self.options.clone(),
+        }
+    }
+}
+impl Default for MarkdonwItOptions {
+    fn default() -> Self {
+        Self {
+            options: HashMap::new(),
+        }
+    }
+}
+
+impl MarkdonwItOptions {
+    pub fn new(options: HashMap<String, String>) -> Self {
+        Self { options }
+    }
+
+    pub fn add(self, md: &mut MarkdownIt) {
+        md.ext.insert::<Self>(self);
+    }
+
+    pub fn is_enabled(&self, key: &str, default: bool) -> bool {
+        match self.get_option_or_default(key, "_NOT_APPLICABLE").as_str() {
+            "true" => true,
+            "false" => false,
+            _ => default,
+        }
+    }
+
+    pub fn get_option(&self, key: &str) -> Option<&String> {
+        self.options.get(key)
+    }
+
+    pub fn get_option_or_default(&self, key: &str, default: &str) -> String {
+        self.options
+            .get(key)
+            .cloned()
+            .unwrap_or_else(|| default.to_string())
+    }
+
+    pub fn internal_domain(&self) -> Option<InternalDomain> {
+        let domain_name = self.get_option("internal_domain_name");
+        match domain_name {
+            None => None,
+            Some(domain_name) => InternalDomain::new(domain_name.clone()),
+        }
+    }
+}
+
+impl InternalDomain {
+    fn new(domain_name: String) -> Option<Self> {
+        if let Ok(base_url) = Url::parse(domain_name.as_str()) {
+            if base_url.domain().is_some() {
+                return Some(InternalDomain { base_url });
+            }
+        }
+
+        let domain_with_scheme = format!("https://{}", domain_name);
+        if let Ok(base_url) = Url::parse(&domain_with_scheme) {
+            if base_url.domain().is_some() {
+                return Some(InternalDomain { base_url });
+            }
+        }
+
+        None
+    }
+
+    pub fn matches(&self, url: &str) -> bool {
+        // let url_parser = Url::options().base_url(Some(&self.base_url));
+        let internal_domain_name = self.base_url.domain().unwrap();
+
+        // url with scheme
+        if Self::check_scheme(url) {
+            return self.check_internal_domain(url, internal_domain_name, false);
+        }
+
+        // 一部の「schemeなし、パス付きURL表記」をヒューリスティックに判定できるが
+        // 仕様が複雑になるためいったんやらない
+        // （google.com/foo は外部サイトとして判定しうるが、google.com は foo.pdf
+        // と判別がつかないため内部リンク扱いとなり、ルールがわかりにくくなるため）
+        // // possiblly a domain name
+        // // heuristics: if the url contains a dot, it's a relative path with domain
+        // // we do not regard them as internal links
+        // // e.g. google.com/foo/bar
+        // let domain_regex = regex!(r"^[a-zA-Z0-9-]+(\.[a-zA-Z0-9-]+)+/");
+        // if domain_regex.is_match(url) {
+        //     println!(
+        //         "check #2 - url: {}, internal_domain_name: {}",
+        //         url, internal_domain_name
+        //     );
+        //     return false;
+        // }
+
+        // should be a relative path
+        // NOTE: google.com（外部サイト）, google.pdf（ファイル名）は区別がつかないため全て内部リンク扱いとする
+        // 外部サイトの場合はスキームをつけること
+        self.check_internal_domain(&url, internal_domain_name, true)
+    }
+
+    fn check_scheme(url: &str) -> bool {
+        if let Ok(url) = Url::parse(url) {
+            if !url.scheme().is_empty() {
+                return true;
+            }
+        }
+        false
+    }
+
+    fn check_internal_domain(
+        &self,
+        url: &str,
+        internal_domain_name: &str,
+        is_base_url_required: bool,
+    ) -> bool {
+        let url = if is_base_url_required {
+            Url::options().base_url(Some(&self.base_url)).parse(url)
+        } else {
+            Url::parse(url)
+        };
+        if let Ok(url) = url {
+            if let Some(domain) = url.domain() {
+                // for `example.com`, `foo.example.com` is also considered as an internal link
+                if domain == internal_domain_name
+                    || domain.ends_with(format!(".{}", internal_domain_name).as_str())
+                {
+                    return true;
+                }
+            }
+        }
+        false
+    }
+}
+
+#[test]
+fn test_get_option() {
+    let raw_options = HashMap::from([
+        ("some_random_option".to_string(), "true".to_string()),
+        ("another_option".to_string(), "some-string".to_string()),
+    ]);
+    let options = MarkdonwItOptions::new(raw_options);
+
+    assert_eq!(
+        options.get_option("some_random_option"),
+        Some(&"true".to_string())
+    );
+    assert_eq!(
+        options.get_option("another_option"),
+        Some(&"some-string".to_string())
+    );
+}
+
+#[test]
+fn test_get_option_or_default() {
+    let raw_options = HashMap::from([("some_random_option".to_string(), "true".to_string())]);
+    let options = MarkdonwItOptions::new(raw_options);
+
+    assert_eq!(
+        options.get_option_or_default("some_random_option", "false"),
+        "true".to_string()
+    );
+    assert_eq!(
+        options.get_option_or_default("another_option", "default value"),
+        "default value".to_string()
+    );
+}
+
+#[test]
+fn test_is_enabled() {
+    {
+        // with values set to "true"
+        let raw_options = HashMap::from([("some_option".to_string(), "true".to_string())]);
+        let options = MarkdonwItOptions::new(raw_options);
+        assert_eq!(options.is_enabled("some_option", false), true);
+        assert_eq!(options.is_enabled("some_option", true), true);
+    }
+
+    {
+        // with values set to "false"
+        let raw_options = HashMap::from([("some_option".to_string(), "false".to_string())]);
+        let options = MarkdonwItOptions::new(raw_options);
+        assert_eq!(options.is_enabled("some_option", false), false);
+        assert_eq!(options.is_enabled("some_option", true), false);
+    }
+
+    {
+        // with values set to other than "true" or "false"
+        let raw_options =
+            HashMap::from([("some_option".to_string(), "some_random-value".to_string())]);
+        let options = MarkdonwItOptions::new(raw_options);
+        assert_eq!(options.is_enabled("some_option", false), false);
+        assert_eq!(options.is_enabled("some_option", true), true);
+    }
+
+    {
+        // with no value set
+        let options = MarkdonwItOptions::new(HashMap::new());
+        assert_eq!(options.is_enabled("some_option", false), false);
+        assert_eq!(options.is_enabled("some_option", true), true);
+    }
+}
+
+#[test]
+fn test_internal_domain_matches() {
+    {
+        // with base_url that has scheme part
+        let base_url = "https://shizuoka.jp".to_string();
+        let internal_domain = InternalDomain::new(base_url).unwrap();
+
+        {
+            // when the url shares the same domain
+            assert_eq!(internal_domain.matches("https://shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("https://fuji.shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("http://fuji.shizuoka.jp"), true);
+            assert_eq!(
+                internal_domain.matches("https://www.city.fuji.shizuoka.jp"),
+                true
+            );
+        }
+
+        {
+            // when the url does not share the same domain
+            assert_eq!(internal_domain.matches("https://fuji.shizu-oka.jp"), false);
+        }
+
+        {
+            // when the url does not have a scheme part
+            assert_eq!(internal_domain.matches("fuji.shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("www.city.fuji.shizuoka.jp"), true);
+            assert_eq!(
+                internal_domain.matches("totally-different-domain.com"),
+                true
+            );
+        }
+
+        {
+            //with relative path
+            assert_eq!(internal_domain.matches("/foo/bar"), true);
+            assert_eq!(internal_domain.matches("/foo/bar/baz.jpg"), true);
+        }
+    }
+
+    {
+        // with base_url that does NOT have scheme part
+        let base_url = "shizuoka.jp".to_string();
+        let internal_domain = InternalDomain::new(base_url).unwrap();
+
+        {
+            // when the url shares the same domain
+            assert_eq!(internal_domain.matches("https://shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("https://fuji.shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("http://fuji.shizuoka.jp"), true);
+            assert_eq!(
+                internal_domain.matches("https://www.city.fuji.shizuoka.jp"),
+                true
+            );
+        }
+
+        {
+            // when the url does not share the same domain
+            assert_eq!(internal_domain.matches("https://fuji.shizu-oka.jp"), false);
+        }
+
+        {
+            // when the url does not have a scheme part
+            assert_eq!(internal_domain.matches("fuji.shizuoka.jp"), true);
+            assert_eq!(internal_domain.matches("www.city.fuji.shizuoka.jp"), true);
+            assert_eq!(
+                internal_domain.matches("totally-different-domain.com"),
+                true
+            );
+        }
+
+        {
+            //with relative path
+            assert_eq!(internal_domain.matches("/foo/bar"), true);
+            assert_eq!(internal_domain.matches("/foo/bar/baz.jpg"), true);
+        }
+    }
+}

data/ext/markdown_it_ruby/src/driver.rs

@@ -0,0 +1,56 @@
+mod options;
+
+use crate::extensions;
+use markdown_it::plugins::{cmark, extra, html};
+use markdown_it::{MarkdownIt, Node};
+pub use options::{InternalDomain, MarkdonwItOptions};
+use std::collections::HashMap;
+use std::sync::OnceLock;
+
+pub(super) struct MarkdownDriver {
+    md: MarkdownIt,
+    source: OnceLock<String>,
+    contents: OnceLock<Node>,
+}
+
+impl MarkdownDriver {
+    pub(super) fn new(env: HashMap<String, String>) -> Self {
+        // create markdown parser
+        let mut md = MarkdownIt::new();
+        let option = MarkdonwItOptions::new(env.clone());
+        Self::prepare(&mut md, option);
+
+        Self {
+            md,
+            source: OnceLock::new(),
+            contents: OnceLock::new(),
+        }
+    }
+
+    pub(super) fn parse(&self, contents: String) {
+        if self.source.set(contents.clone()).is_err() {
+            return;
+        }
+
+        let root = self.md.parse(contents.as_str());
+        self.contents.set(root).unwrap();
+    }
+
+    pub(super) fn render(&self) -> String {
+        let contents = self.contents.get();
+        match contents {
+            None => String::new(),
+            Some(contents) => contents.render(),
+        }
+    }
+
+    fn prepare(md: &mut MarkdownIt, option: MarkdonwItOptions) {
+        html::add(md);
+        cmark::add(md);
+        extra::add(md);
+
+        // add custom three rules described above
+        extensions::add(md, &option);
+        option.add(md);
+    }
+}

data/ext/markdown_it_ruby/src/extensions/heading_level_modification.rs

@@ -0,0 +1,139 @@
+use crate::driver::MarkdonwItOptions;
+use markdown_it::parser::core::CoreRule;
+use markdown_it::plugins::cmark::block::heading::ATXHeading;
+use markdown_it::{MarkdownIt, Node, NodeValue, Renderer};
+
+#[derive(Debug)]
+struct PlainTextElement {
+    text: String,
+}
+
+impl NodeValue for PlainTextElement {
+    fn render(&self, node: &Node, fmt: &mut dyn Renderer) {
+        fmt.open("p", &[]);
+        fmt.text(&self.text);
+        fmt.contents(&node.children);
+        fmt.close("p");
+        fmt.cr();
+    }
+}
+
+struct HeadingLevelModificationRule;
+
+impl CoreRule for HeadingLevelModificationRule {
+    fn run(root: &mut Node, md: &MarkdownIt) {
+        let options = md.ext.get::<MarkdonwItOptions>();
+        let heading_level_offset = match options {
+            Some(options) => options
+                .get_option_or_default("heading_level_offset", "0")
+                .parse::<u8>()
+                .unwrap_or(0),
+            None => 0,
+        };
+        // walk through AST recursively and count the number of two
+        // custom nodes added by other two rules
+        root.walk_mut(|node, _| {
+            if let Some(heading) = node.cast::<ATXHeading>() {
+                // do not render level 1 and level > 4 headings (# and ####+)
+                // illegal `headings` has to be treated as plain text
+                let new_heading_level = heading.level + heading_level_offset;
+                // if the new heading level is out of the accepted range (1-6), treat it as plain text
+                if new_heading_level < 1 || new_heading_level > 6 {
+                    node.replace(PlainTextElement {
+                        text: "#".repeat(heading.level as usize) + " ",
+                    });
+                } else {
+                    node.replace(ATXHeading {
+                        level: new_heading_level,
+                    });
+                }
+            }
+        });
+    }
+}
+
+pub fn add(md: &mut MarkdownIt) {
+    md.add_rule::<HeadingLevelModificationRule>();
+}
+
+#[test]
+fn test_heading_modification() {
+    use std::collections::HashMap;
+
+    let mut md = MarkdownIt::new();
+    markdown_it::plugins::cmark::add(&mut md);
+    add(&mut md);
+
+    {
+        // without options
+        {
+            // for heading levels out of the accepted range (h7)
+            let src = "####### heading 7";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<p>####### heading 7</p>\n");
+        }
+
+        {
+            // for heading levels within the accepted range(h1-h6)
+            let src = "# heading 1";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h1>heading 1</h1>\n");
+
+            let src = "### heading 3";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h3>heading 3</h3>\n");
+
+            let src = "###### heading 6";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h6>heading 6</h6>\n");
+        }
+
+        let src =
+            "# heading 1\n## heading 2\n### heading 3\n#### heading 4\n##### heading 5\n###### heading 6\n####### heading 7";
+        let html = md.parse(src).render();
+
+        assert_eq!(
+            html,
+            "<h1>heading 1</h1>\n<h2>heading 2</h2>\n<h3>heading 3</h3>\n<h4>heading 4</h4>\n<h5>heading 5</h5>\n<h6>heading 6</h6>\n<p>####### heading 7</p>\n"
+            );
+    }
+
+    {
+        // with options
+        let options = MarkdonwItOptions::new(HashMap::from([(
+            "heading_level_offset".to_string(),
+            "3".to_string(),
+        )]));
+        options.add(&mut md);
+        {
+            // for heading levels out of the accepted range (h4 - shifted to h7)
+            let src = "#### heading 4";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<p>#### heading 4</p>\n");
+        }
+
+        {
+            // for heading levels within the accepted range(h1-h3)
+            let src = "# heading 1";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h4>heading 1</h4>\n");
+
+            let src = "## heading 2";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h5>heading 2</h5>\n");
+
+            let src = "### heading 3";
+            let html = md.parse(src).render();
+            assert_eq!(html, "<h6>heading 3</h6>\n");
+        }
+
+        let src =
+            "# heading 1\n## heading 2\n### heading 3\n#### heading 4\n##### heading 5\n###### heading 6\n####### heading 7";
+        let html = md.parse(src).render();
+
+        assert_eq!(
+            html,
+            "<h4>heading 1</h4>\n<h5>heading 2</h5>\n<h6>heading 3</h6>\n<p>#### heading 4</p>\n<p>##### heading 5</p>\n<p>###### heading 6</p>\n<p>####### heading 7</p>\n"
+            );
+    }
+}