kramdown-syntax_tree_sitter 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02e58047b1cac1778ad13d25f88a64a42f0714faeba928322851a78afeb4ce66
-  data.tar.gz: 87cc78c75fb83a9a56cca8bb663f72b1d3abbd03f3920e37bad6ac629c699d69
+  metadata.gz: 4ba9a8e090021116214ebf5938b8de67856e7aa0054aeb49de5a67041b0683a5
+  data.tar.gz: b9d7a9110a29f9abd175ea82c3f54eccb15ca334b69181299f291f43c8c6c35b
 SHA512:
-  metadata.gz: 3b609ade16479c2e09504248970b1f95eb496b982cc641e766881dbfc74bedd580a45bc5f0c24f9b859535a3967a74e51126e6dfdf8df7448a5284da259e3105
-  data.tar.gz: ae2cdab47a180b7435fc9c01bff81aefb377a353af341dc98f3f5bd8685105f6b976d35c25a900fcc7caa8744e3c34ef3fdbb63fb09dc27d52ee63f67644466e
+  metadata.gz: ff14f9dff79b8681771b4a0b078e8a1a820489cb2e338f91f5f2ca30d758495a19763f9449585429872c00943c38dca10e88a9465eb7a97cd1c64c298935d528
+  data.tar.gz: fc629b06c06b8df3614084dd40ccf8d38747ddc5ab9e096cb85f5976feb0a8f0729615e303a88f256b0bdd9be0c96309ec2cd461bedf472bf7e2f7aa5d4605a5

data/README.md

@@ -1,5 +1,8 @@
 # Kramdown Tree-sitter Highlighter
 
+[![Build status](https://img.shields.io/github/workflow/status/andrewtbiehl/kramdown-syntax_tree_sitter/Quality%20Control?style=flat-square&logo=github)](https://github.com/andrewtbiehl/kramdown-syntax_tree_sitter/actions/workflows/quality-control.yml)
+[![RubyGems page](https://img.shields.io/gem/v/kramdown-syntax_tree_sitter?style=flat-square&color=blue&logo=rubygems&logoColor=white)](https://rubygems.org/gems/kramdown-syntax_tree_sitter)
+
 ***Syntax highlight code with [Tree-sitter](https://tree-sitter.github.io/tree-sitter)
 via [Kramdown](https://kramdown.gettalong.org).***
 
@@ -29,14 +32,13 @@ For projects using [Bundler](https://bundler.io) for dependency management, run
 following command to both install the gem and add it to the Gemfile:
 
 ```shell
-bundle add kramdown-syntax_tree_sitter --github andrewtbiehl/kramdown-syntax_tree_sitter
+bundle add kramdown-syntax_tree_sitter
 ```
 
-Otherwise, download this project's repository and then run the following command from
-within it to build and install the gem:
+Otherwise, install the gem via the following command:
 
 ```shell
-gem build && gem install kramdown-syntax_tree_sitter
+gem install kramdown-syntax_tree_sitter
 ```
 
 ## Usage
@@ -176,14 +178,58 @@ block will only be correctly highlighted if the language identifier provided for
 code block is its language's corresponding Tree-sitter scope string. This is illustrated
 by the code block used in the [Quickstart](#quickstart) example.
 
+### CSS styling
+
+Code highlights can be rendered either via inline CSS styles or via CSS classes, which
+are applied to each token in the parsed code.
+
+To use CSS classes for highlighting, set the `css_classes` Kramdown syntax-highlighting
+option to `true`. Otherwise, the plugin will apply highlights via inline CSS styles.
+
+The inline CSS styles are derived from Tree-sitter's built-in default highlighting
+theme, repeated here for convenience:
+
+| Token name | CSS style |
+| :-- | :-- |
+| attribute | `color: #af0000; font-style: italic;` |
+| comment | `color: #8a8a8a; font-style: italic;` |
+| constant | `color: #875f00;` |
+| function | `color: o#005fd7;` |
+| function.builtin | `color: #005fd7; font-weight: bold;` |
+| keyword | `color: #5f00d7;` |
+| operator | `color: #4e4e4e; font-weight: bold;` |
+| property | `color: #af0000;` |
+| string | `color: #008700;` |
+| string.special | `color: #008787;` |
+| tag | `color: #000087;` |
+| type | `color: #005f5f;` |
+| type.builtin | `color: #005f5f; font-weight: bold;` |
+| variable.builtin | `font-weight: bold;` |
+| variable.parameter | `text-decoration: underline;` |
+| constant.builtin, number | `color: #875f00; font-weight: bold;` |
+| constructor, module | `color: #af8700;` |
+| punctuation.bracket, punctuation.delimiter | `color: #4e4e4e;` |
+
+Any and all token types not represented in this default theme are consequently not
+highlighted when using the inline CSS styles option.
+
+The CSS class names are derived directly from Tree-sitter token names by replacing all
+full stops ('.') with dashes ('-') and adding the prefix 'ts-'. For example, the CSS
+class for 'function.builtin' tokens is 'ts-function-builtin'. The use of CSS classes
+allows for customization of highlighting styles, including the ability to highlight more
+token types than with the default inline CSS method. Of course, this also requires that
+an externally created CSS stylesheet defining the style for each token type is provided
+whenever the Kramdown-generated HTML is rendered.
+
 ### Configuration
 
 This Kramdown plugin currently supports the following options when provided as sub-keys
 of the Kramdown option `syntax_highlighter_opts`:
 
-| Key | Description | Default value |
-| :-- | :-- | :-- |
-| `tree_sitter_parsers_dir` | The path to the Tree-sitter language parsers directory. | `~/tree_sitter_parsers` |
+| Key | Description | Type | Default value |
+| :-- | :-- | :-- | :-- |
+| `tree_sitter_parsers_dir` | The path to the Tree-sitter language parsers directory. | String | `~/tree_sitter_parsers` |
+| `css_classes` | Whether to use CSS classes for highlights. | Boolean | `false` |
 
 ## Contributing
 

data/ext/tree_sitter_adapter/src/lib.rs

@@ -1,21 +1,28 @@
 #[macro_use]
 extern crate rutie;
 
-use rutie::{AnyException, Class, Exception, Object, RString, VM};
+use rutie::{AnyException, Boolean, Class, Exception, Object, RString, VM};
 
 mod tree_sitter_adapter;
 
 class!(TreeSitterAdapter);
 
+#[rustfmt::skip]
 methods!(
     TreeSitterAdapter,
     _rtself,
-    fn pub_highlight(raw_code: RString, raw_parsers_dir: RString, raw_scope: RString) -> RString {
+    fn pub_highlight(
+        raw_code: RString,
+        raw_parsers_dir: RString,
+        raw_scope: RString,
+        css_classes: Boolean
+    ) -> RString {
         VM::unwrap_or_raise_ex(
             tree_sitter_adapter::highlight(
                 &raw_code.unwrap().to_string(),
                 &raw_parsers_dir.unwrap().to_string(),
                 &raw_scope.unwrap().to_string(),
+                css_classes.unwrap().to_bool(),
             )
             .as_ref()
             .map(String::as_str)

data/ext/tree_sitter_adapter/src/tree_sitter_adapter.rs

@@ -1,7 +1,9 @@
-use anyhow::{Context, Error, Result};
+use anyhow::{Context, Result};
+use std::collections::HashMap;
 use std::convert;
 use std::path::PathBuf;
 use tree_sitter::Language;
+use tree_sitter_cli::highlight::Style;
 use tree_sitter_cli::highlight::Theme;
 use tree_sitter_highlight::{Error as TSError, HighlightEvent};
 use tree_sitter_highlight::{Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
@@ -9,126 +11,93 @@ use tree_sitter_loader::{Config, LanguageConfiguration, Loader};
 
 const LOADER_ERROR_MSG: &str = "Error loading Tree-sitter parsers from directory";
 const NO_LANGUAGE_ERROR_MSG: &str = "Error retrieving language configuration for scope";
+const NO_HIGHLIGHT_ERROR_MSG: &str = "Error retrieving highlight configuration for scope";
 
-pub fn highlight(code: &str, parsers_dir: &str, scope: &str) -> Result<String, String> {
-    let parsers_dir = PathBuf::from(parsers_dir);
-    let theme = Theme::default();
-    let mut loader = Loader::new_from_dir(parsers_dir).map_err(Error::to_formatted_string)?;
-    loader.configure_highlights(&theme.highlight_names);
-    loader
-        .language_configuration_from_scope(scope)
-        .and_then(LanguageConfigurationAdapter::highlight_config)
-        .and_then(|config| {
-            let default_css_style = String::new();
-            let css_attribute_callback = get_css_styles(&theme, &default_css_style);
-            HighlighterAdapter::new(&loader, config)
-                .highlight(code)
-                .and_then(|highlights| render_html(highlights, &css_attribute_callback))
-        })
-        .map_err(Error::to_formatted_string)
-}
-
-trait LoaderExt {
-    fn new_from_dir(parser_directory: PathBuf) -> Result<Loader>;
-
-    fn language_configuration_from_scope<'a>(
-        &'a self,
-        scope: &'a str,
-    ) -> Result<LanguageConfigurationAdapter<'a>>;
+trait ResultExt<T, E> {
+    fn flatten_(self) -> Result<T, E>;
 }
 
-impl LoaderExt for Loader {
-    fn new_from_dir(parser_directory: PathBuf) -> Result<Loader> {
-        Loader::new()
-            .and_then(|mut loader| {
-                let config = {
-                    let parser_directory = parser_directory.clone();
-                    let parser_directories = vec![parser_directory];
-                    Config { parser_directories }
-                };
-                loader.find_all_languages(&config)?;
-                Ok(loader)
-            })
-            .with_context(|| {
-                let parser_directory_string = parser_directory.display();
-                format!("{LOADER_ERROR_MSG} '{parser_directory_string}'")
-            })
-    }
-
-    fn language_configuration_from_scope<'a>(
-        &'a self,
-        scope: &'a str,
-    ) -> Result<LanguageConfigurationAdapter<'a>> {
-        self.language_configuration_for_scope(scope)
-            .transpose()
-            .context("Language not found")
-            .flatten_()
-            .map(|(language, config)| LanguageConfigurationAdapter { language, config })
-            .with_context(|| format!("{NO_LANGUAGE_ERROR_MSG} '{scope}'"))
+impl<T, E> ResultExt<T, E> for Result<Result<T, E>, E> {
+    fn flatten_(self) -> Result<T, E> {
+        self.and_then(convert::identity)
     }
 }
 
-struct LanguageConfigurationAdapter<'a> {
-    language: Language,
-    config: &'a LanguageConfiguration<'a>,
+fn loader(parser_directory: PathBuf) -> Result<Loader> {
+    Loader::new()
+        .and_then(|mut loader| {
+            let config = {
+                let parser_directory = parser_directory.clone();
+                let parser_directories = vec![parser_directory];
+                Config { parser_directories }
+            };
+            loader.find_all_languages(&config)?;
+            Ok(loader)
+        })
+        .with_context(|| {
+            let parser_directory_str = parser_directory.display();
+            format!("{LOADER_ERROR_MSG} '{parser_directory_str}'")
+        })
 }
 
-impl<'a> LanguageConfigurationAdapter<'a> {
-    fn highlight_config(self) -> Result<&'a HighlightConfiguration> {
-        self.config
-            .highlight_config(self.language)
-            .transpose()
-            .context("Another issue")
-            .flatten_()
-    }
+fn language_and_configuration<'a>(
+    loader: &'a Loader,
+    scope: &'a str,
+) -> Result<(Language, &'a LanguageConfiguration<'a>)> {
+    loader
+        .language_configuration_for_scope(scope)
+        .transpose()
+        .context("Language not found")
+        .flatten_()
+        .with_context(|| format!("{NO_LANGUAGE_ERROR_MSG} '{scope}'"))
 }
 
-struct HighlightsAdapter<'a, T: Iterator<Item = Result<HighlightEvent, TSError>>> {
-    code: &'a str,
-    highlights: T,
+fn highlight_configuration<'a>(
+    language: Language,
+    config: &'a LanguageConfiguration<'a>,
+    scope: &'a str,
+) -> Result<&'a HighlightConfiguration> {
+    config
+        .highlight_config(language)
+        .transpose()
+        .with_context(|| format!("{NO_HIGHLIGHT_ERROR_MSG} '{scope}'"))
+        .flatten_()
 }
 
-struct HighlighterAdapter<'a> {
-    loader: &'a Loader,
-    config: &'a HighlightConfiguration,
-    highlighter: Highlighter,
+fn highlights(
+    code: &str,
+    config: &HighlightConfiguration,
+    loader: &Loader,
+) -> Result<impl Iterator<Item = Result<HighlightEvent, TSError>>> {
+    Highlighter::new()
+        .highlight(config, code.as_bytes(), None, |s| {
+            loader.highlight_config_for_injection_string(s)
+        })
+        .map(Iterator::collect)
+        .map(Vec::into_iter)
+        .map_err(Into::into)
 }
 
-impl<'a> HighlighterAdapter<'a> {
-    fn new(loader: &'a Loader, config: &'a HighlightConfiguration) -> Self {
-        Self {
-            loader,
-            config,
-            highlighter: Highlighter::new(),
-        }
-    }
-
-    fn highlight(
-        &'a mut self,
-        code: &'a str,
-    ) -> Result<HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>>
-    {
-        let Self {
-            loader,
-            config,
-            highlighter,
-        } = self;
-        highlighter
-            .highlight(config, code.as_bytes(), None, |s| {
-                loader.highlight_config_for_injection_string(s)
-            })
-            .map(|highlights| HighlightsAdapter { code, highlights })
-            .map_err(Into::into)
+fn create_html_attribute_callback<'a>(
+    html_attributes: &'a [String],
+) -> impl Fn(Highlight) -> &'a [u8] {
+    |highlight| {
+        html_attributes
+            .get(highlight.0)
+            .map(String::as_str)
+            .unwrap_or_default()
+            .as_bytes()
     }
 }
 
-fn render_html<'a, F: Fn(Highlight) -> &'a [u8]>(
-    highlights: HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>,
-    css_attribute_callback: &F,
+fn render_html(
+    code: &str,
+    highlights: impl Iterator<Item = Result<HighlightEvent, TSError>>,
+    html_attributes: &[String],
 ) -> Result<String> {
-    let HighlightsAdapter { code, highlights } = highlights;
+    let html_attribute_callback = create_html_attribute_callback(html_attributes);
     let mut renderer = HtmlRenderer::new();
-    renderer.render(highlights, code.as_bytes(), css_attribute_callback)?;
+    renderer.render(highlights, code.as_bytes(), &html_attribute_callback)?;
     // Remove erroneously appended newline
     if renderer.html.ends_with(&[b'\n']) && !code.ends_with('\n') {
         renderer.html.pop();
@@ -136,36 +105,65 @@ fn render_html<'a, F: Fn(Highlight) -> &'a [u8]>(
     Ok(renderer.lines().collect())
 }
 
-fn get_css_styles<'a>(
-    theme: &'a Theme,
-    default_css_style: &'a String,
-) -> impl Fn(Highlight) -> &'a [u8] {
-    |highlight| {
-        theme
-            .styles
-            .get(highlight.0)
-            .and_then(|style| style.css.as_ref())
-            .unwrap_or(default_css_style)
-            .as_bytes()
-    }
+fn highlight_names(scope: &str, loader: &Loader) -> Result<Vec<String>> {
+    let (language, config) = language_and_configuration(loader, scope)?;
+    let highlight_config = highlight_configuration(language, config, scope)?;
+    Ok(highlight_config.names().iter().map(String::from).collect())
 }
 
-trait ResultExt<T, E> {
-    fn flatten_(self) -> Result<T, E>;
+fn highlight_name_styles() -> HashMap<String, Style> {
+    let theme = Theme::default();
+    theme
+        .highlight_names
+        .into_iter()
+        .zip(theme.styles.into_iter())
+        .collect()
 }
 
-impl<T, E> ResultExt<T, E> for Result<Result<T, E>, E> {
-    fn flatten_(self) -> Result<T, E> {
-        self.and_then(convert::identity)
-    }
+fn inline_css_attributes(highlight_names: &[String]) -> Vec<String> {
+    let highlight_name_styles = highlight_name_styles();
+    highlight_names
+        .iter()
+        .map(|n| highlight_name_styles.get(n))
+        .map(|o| o.and_then(|style| style.css.as_ref()))
+        .map(|o| o.map(String::from))
+        .map(Option::unwrap_or_default)
+        .collect()
 }
 
-trait ErrorExt {
-    fn to_formatted_string(self) -> String;
+fn css_class_attributes(highlight_names: &[String]) -> Vec<String> {
+    highlight_names
+        .iter()
+        .map(|s| s.replace('.', "-"))
+        .map(|s| format!("class='ts-{s}'"))
+        .collect()
 }
 
-impl ErrorExt for Error {
-    fn to_formatted_string(self) -> String {
-        format!("{self:#}")
-    }
+fn highlight_adapter(
+    code: &str,
+    parsers_dir: &str,
+    scope: &str,
+    css_classes: bool,
+) -> Result<String> {
+    let parsers_dir = PathBuf::from(parsers_dir);
+    let mut loader = loader(parsers_dir)?;
+    let highlight_names = highlight_names(scope, &loader)?;
+    loader.configure_highlights(&highlight_names);
+    let (language, config) = language_and_configuration(&loader, scope)?;
+    let highlight_config = highlight_configuration(language, config, scope)?;
+    let highlights = highlights(code, highlight_config, &loader)?;
+    let html_attributes = match css_classes {
+        true => css_class_attributes,
+        false => inline_css_attributes,
+    }(&highlight_names);
+    render_html(code, highlights, &html_attributes)
+}
+
+pub fn highlight(
+    code: &str,
+    parsers_dir: &str,
+    scope: &str,
+    css_classes: bool,
+) -> Result<String, String> {
+    highlight_adapter(code, parsers_dir, scope, css_classes).map_err(|e| format!("{e:#}"))
 }

data/lib/kramdown/converter/syntax_highlighter/tree_sitter.rb

@@ -16,14 +16,26 @@ module Kramdown
         def self.call(converter, raw_text, language, type, _)
           return nil unless language
 
-          parsers_dir = get_option(converter, :tree_sitter_parsers_dir)
-                        .then { _1 || DEFAULT_PARSERS_DIR }
-                        .then { File.expand_path _1 }
-          rendered_text = TreeSitterAdapter.highlight raw_text, parsers_dir, language
+          rendered_text = TreeSitterAdapter.highlight(
+            raw_text,
+            get_parsers_dir(converter),
+            language,
+            get_use_css_classes(converter)
+          )
           # Code blocks are additionally wrapped in HTML code tags
           type == :block ? "<pre><code>#{rendered_text}</code></pre>" : rendered_text
         end
 
+        def self.get_parsers_dir(converter)
+          File.expand_path(
+            get_option(converter, :tree_sitter_parsers_dir) || DEFAULT_PARSERS_DIR
+          )
+        end
+
+        def self.get_use_css_classes(converter)
+          get_option(converter, :css_classes) || false
+        end
+
         def self.get_option(converter, name)
           converter.options[:syntax_highlighter_opts][name]
         end

data/lib/kramdown/syntax_tree_sitter/version.rb

@@ -5,7 +5,7 @@ module Kramdown
     module SyntaxHighlighter
       module TreeSitter
         # Version of kramdown-syntax_tree_sitter gem
-        VERSION = '0.1.0'
+        VERSION = '0.2.0'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kramdown-syntax_tree_sitter
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew T. Biehl
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-11 00:00:00.000000000 Z
+date: 2022-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kramdown