kramdown-syntax_tree_sitter 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,16 @@
1
+ [package]
2
+ name = "tree_sitter_adapter"
3
+ version = "0.0.0"
4
+ edition = "2021"
5
+
6
+ [lib]
7
+ name = "tree_sitter_adapter"
8
+ crate-type = ["cdylib"]
9
+
10
+ [dependencies]
11
+ anyhow = "*"
12
+ tree-sitter = "*"
13
+ tree-sitter-cli = "*"
14
+ tree-sitter-highlight = "*"
15
+ tree-sitter-loader = "*"
16
+ rutie = "*"
@@ -0,0 +1,62 @@
1
+ #[macro_use]
2
+ extern crate rutie;
3
+
4
+ use rutie::{AnyException, Class, Exception, Object, RString, VM};
5
+
6
+ mod tree_sitter_adapter;
7
+
8
+ class!(TreeSitterAdapter);
9
+
10
+ methods!(
11
+ TreeSitterAdapter,
12
+ _rtself,
13
+ fn pub_highlight(raw_code: RString, raw_parsers_dir: RString, raw_scope: RString) -> RString {
14
+ VM::unwrap_or_raise_ex(
15
+ tree_sitter_adapter::highlight(
16
+ &raw_code.unwrap().to_string(),
17
+ &raw_parsers_dir.unwrap().to_string(),
18
+ &raw_scope.unwrap().to_string(),
19
+ )
20
+ .as_ref()
21
+ .map(String::as_str)
22
+ .map(RString::new_utf8)
23
+ .map_err(String::as_str)
24
+ .map_err(AnyException::new_runtime_error),
25
+ )
26
+ }
27
+ );
28
+
29
+ #[no_mangle]
30
+ pub extern "C" fn Init_tree_sitter_adapter() {
31
+ Class::new("TreeSitterAdapter", None).define(|class_| {
32
+ class_.def_self("highlight", pub_highlight);
33
+ });
34
+ }
35
+
36
+ pub trait VMExt {
37
+ fn unwrap_or_raise_ex<T, E>(x: Result<T, E>) -> T
38
+ where
39
+ E: Into<AnyException>;
40
+ }
41
+
42
+ impl VMExt for VM {
43
+ fn unwrap_or_raise_ex<T, E>(result: Result<T, E>) -> T
44
+ where
45
+ E: Into<AnyException>,
46
+ {
47
+ result.unwrap_or_else(|e| {
48
+ VM::raise_ex(e);
49
+ unreachable!();
50
+ })
51
+ }
52
+ }
53
+
54
+ pub trait AnyExceptionExt {
55
+ fn new_runtime_error(message: &str) -> Self;
56
+ }
57
+
58
+ impl AnyExceptionExt for AnyException {
59
+ fn new_runtime_error(message: &str) -> Self {
60
+ AnyException::new("RuntimeError", Some(message))
61
+ }
62
+ }
@@ -0,0 +1,171 @@
1
+ use anyhow::{Context, Error, Result};
2
+ use std::convert;
3
+ use std::path::PathBuf;
4
+ use tree_sitter::Language;
5
+ use tree_sitter_cli::highlight::Theme;
6
+ use tree_sitter_highlight::{Error as TSError, HighlightEvent};
7
+ use tree_sitter_highlight::{Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
8
+ use tree_sitter_loader::{Config, LanguageConfiguration, Loader};
9
+
10
+ const LOADER_ERROR_MSG: &str = "Error loading Tree-sitter parsers from directory";
11
+ const NO_LANGUAGE_ERROR_MSG: &str = "Error retrieving language configuration for scope";
12
+
13
+ pub fn highlight(code: &str, parsers_dir: &str, scope: &str) -> Result<String, String> {
14
+ let parsers_dir = PathBuf::from(parsers_dir);
15
+ let theme = Theme::default();
16
+ let mut loader = Loader::new_from_dir(parsers_dir).map_err(Error::to_formatted_string)?;
17
+ loader.configure_highlights(&theme.highlight_names);
18
+ loader
19
+ .language_configuration_from_scope(scope)
20
+ .and_then(LanguageConfigurationAdapter::highlight_config)
21
+ .and_then(|config| {
22
+ let default_css_style = String::new();
23
+ let css_attribute_callback = get_css_styles(&theme, &default_css_style);
24
+ HighlighterAdapter::new(&loader, config)
25
+ .highlight(code)
26
+ .and_then(|highlights| render_html(highlights, &css_attribute_callback))
27
+ })
28
+ .map_err(Error::to_formatted_string)
29
+ }
30
+
31
+ trait LoaderExt {
32
+ fn new_from_dir(parser_directory: PathBuf) -> Result<Loader>;
33
+
34
+ fn language_configuration_from_scope<'a>(
35
+ &'a self,
36
+ scope: &'a str,
37
+ ) -> Result<LanguageConfigurationAdapter<'a>>;
38
+ }
39
+
40
+ impl LoaderExt for Loader {
41
+ fn new_from_dir(parser_directory: PathBuf) -> Result<Loader> {
42
+ Loader::new()
43
+ .and_then(|mut loader| {
44
+ let config = {
45
+ let parser_directory = parser_directory.clone();
46
+ let parser_directories = vec![parser_directory];
47
+ Config { parser_directories }
48
+ };
49
+ loader.find_all_languages(&config)?;
50
+ Ok(loader)
51
+ })
52
+ .with_context(|| {
53
+ let parser_directory_string = parser_directory.display();
54
+ format!("{LOADER_ERROR_MSG} '{parser_directory_string}'")
55
+ })
56
+ }
57
+
58
+ fn language_configuration_from_scope<'a>(
59
+ &'a self,
60
+ scope: &'a str,
61
+ ) -> Result<LanguageConfigurationAdapter<'a>> {
62
+ self.language_configuration_for_scope(scope)
63
+ .transpose()
64
+ .context("Language not found")
65
+ .flatten_()
66
+ .map(|(language, config)| LanguageConfigurationAdapter { language, config })
67
+ .with_context(|| format!("{NO_LANGUAGE_ERROR_MSG} '{scope}'"))
68
+ }
69
+ }
70
+
71
+ struct LanguageConfigurationAdapter<'a> {
72
+ language: Language,
73
+ config: &'a LanguageConfiguration<'a>,
74
+ }
75
+
76
+ impl<'a> LanguageConfigurationAdapter<'a> {
77
+ fn highlight_config(self) -> Result<&'a HighlightConfiguration> {
78
+ self.config
79
+ .highlight_config(self.language)
80
+ .transpose()
81
+ .context("Another issue")
82
+ .flatten_()
83
+ }
84
+ }
85
+
86
+ struct HighlightsAdapter<'a, T: Iterator<Item = Result<HighlightEvent, TSError>>> {
87
+ code: &'a str,
88
+ highlights: T,
89
+ }
90
+
91
+ struct HighlighterAdapter<'a> {
92
+ loader: &'a Loader,
93
+ config: &'a HighlightConfiguration,
94
+ highlighter: Highlighter,
95
+ }
96
+
97
+ impl<'a> HighlighterAdapter<'a> {
98
+ fn new(loader: &'a Loader, config: &'a HighlightConfiguration) -> Self {
99
+ Self {
100
+ loader,
101
+ config,
102
+ highlighter: Highlighter::new(),
103
+ }
104
+ }
105
+
106
+ fn highlight(
107
+ &'a mut self,
108
+ code: &'a str,
109
+ ) -> Result<HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>>
110
+ {
111
+ let Self {
112
+ loader,
113
+ config,
114
+ highlighter,
115
+ } = self;
116
+ highlighter
117
+ .highlight(config, code.as_bytes(), None, |s| {
118
+ loader.highlight_config_for_injection_string(s)
119
+ })
120
+ .map(|highlights| HighlightsAdapter { code, highlights })
121
+ .map_err(Into::into)
122
+ }
123
+ }
124
+
125
+ fn render_html<'a, F: Fn(Highlight) -> &'a [u8]>(
126
+ highlights: HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>,
127
+ css_attribute_callback: &F,
128
+ ) -> Result<String> {
129
+ let HighlightsAdapter { code, highlights } = highlights;
130
+ let mut renderer = HtmlRenderer::new();
131
+ renderer.render(highlights, code.as_bytes(), css_attribute_callback)?;
132
+ // Remove erroneously appended newline
133
+ if renderer.html.ends_with(&[b'\n']) && !code.ends_with('\n') {
134
+ renderer.html.pop();
135
+ }
136
+ Ok(renderer.lines().collect())
137
+ }
138
+
139
+ fn get_css_styles<'a>(
140
+ theme: &'a Theme,
141
+ default_css_style: &'a String,
142
+ ) -> impl Fn(Highlight) -> &'a [u8] {
143
+ |highlight| {
144
+ theme
145
+ .styles
146
+ .get(highlight.0)
147
+ .and_then(|style| style.css.as_ref())
148
+ .unwrap_or(default_css_style)
149
+ .as_bytes()
150
+ }
151
+ }
152
+
153
+ trait ResultExt<T, E> {
154
+ fn flatten_(self) -> Result<T, E>;
155
+ }
156
+
157
+ impl<T, E> ResultExt<T, E> for Result<Result<T, E>, E> {
158
+ fn flatten_(self) -> Result<T, E> {
159
+ self.and_then(convert::identity)
160
+ }
161
+ }
162
+
163
+ trait ErrorExt {
164
+ fn to_formatted_string(self) -> String;
165
+ }
166
+
167
+ impl ErrorExt for Error {
168
+ fn to_formatted_string(self) -> String {
169
+ format!("{self:#}")
170
+ }
171
+ }
@@ -0,0 +1,35 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'kramdown'
4
+ require 'tree_sitter_adapter'
5
+
6
+ module Kramdown
7
+ module Converter # rubocop:disable Style/Documentation
8
+ module SyntaxHighlighter
9
+ # This highlighter is not yet fully configured to highlight code.
10
+ #
11
+ # Currently it merely escapes the code so that it can be safely inserted into HTML
12
+ # text.
13
+ module TreeSitter
14
+ DEFAULT_PARSERS_DIR = '~/tree_sitter_parsers'
15
+
16
+ def self.call(converter, raw_text, language, type, _)
17
+ return nil unless language
18
+
19
+ parsers_dir = get_option(converter, :tree_sitter_parsers_dir)
20
+ .then { _1 || DEFAULT_PARSERS_DIR }
21
+ .then { File.expand_path _1 }
22
+ rendered_text = TreeSitterAdapter.highlight raw_text, parsers_dir, language
23
+ # Code blocks are additionally wrapped in HTML code tags
24
+ type == :block ? "<pre><code>#{rendered_text}</code></pre>" : rendered_text
25
+ end
26
+
27
+ def self.get_option(converter, name)
28
+ converter.options[:syntax_highlighter_opts][name]
29
+ end
30
+ end
31
+ end
32
+
33
+ add_syntax_highlighter(:'tree-sitter', SyntaxHighlighter::TreeSitter)
34
+ end
35
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Kramdown
4
+ module Converter
5
+ module SyntaxHighlighter
6
+ module TreeSitter
7
+ # Version of kramdown-syntax_tree_sitter gem
8
+ VERSION = '0.1.0'
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,4 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'converter/syntax_highlighter/tree_sitter'
4
+ require_relative 'syntax_tree_sitter/version'
@@ -0,0 +1,8 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'rutie'
4
+
5
+ Rutie.new(:tree_sitter_adapter).init(
6
+ 'Init_tree_sitter_adapter',
7
+ File.expand_path(File.join(__dir__, '..', 'ext', 'tree_sitter_adapter', 'target'))
8
+ )
metadata ADDED
@@ -0,0 +1,168 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: kramdown-syntax_tree_sitter
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Andrew T. Biehl
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2022-11-11 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: kramdown
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '2.0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '2.0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: rake
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '13.0'
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '13.0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rutie
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 0.0.4
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 0.0.4
55
+ - !ruby/object:Gem::Dependency
56
+ name: minitest
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '5.0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '5.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rouge
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '4.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '4.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: rubocop
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.36'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.36'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rubocop-minitest
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '0.22'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '0.22'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rubocop-rake
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.6'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.6'
125
+ description:
126
+ email:
127
+ executables: []
128
+ extensions:
129
+ - ext/Rakefile
130
+ extra_rdoc_files: []
131
+ files:
132
+ - LICENSE.txt
133
+ - README.md
134
+ - ext/Rakefile
135
+ - ext/tasks.rake
136
+ - ext/tree_sitter_adapter/Cargo.lock
137
+ - ext/tree_sitter_adapter/Cargo.toml
138
+ - ext/tree_sitter_adapter/src/lib.rs
139
+ - ext/tree_sitter_adapter/src/tree_sitter_adapter.rs
140
+ - lib/kramdown/converter/syntax_highlighter/tree_sitter.rb
141
+ - lib/kramdown/syntax_tree_sitter.rb
142
+ - lib/kramdown/syntax_tree_sitter/version.rb
143
+ - lib/tree_sitter_adapter.rb
144
+ homepage: https://github.com/andrewtbiehl/kramdown-syntax_tree_sitter
145
+ licenses:
146
+ - MIT
147
+ metadata:
148
+ rubygems_mfa_required: 'true'
149
+ post_install_message:
150
+ rdoc_options: []
151
+ require_paths:
152
+ - lib
153
+ required_ruby_version: !ruby/object:Gem::Requirement
154
+ requirements:
155
+ - - ">="
156
+ - !ruby/object:Gem::Version
157
+ version: '2.7'
158
+ required_rubygems_version: !ruby/object:Gem::Requirement
159
+ requirements:
160
+ - - ">="
161
+ - !ruby/object:Gem::Version
162
+ version: '0'
163
+ requirements: []
164
+ rubygems_version: 3.3.7
165
+ signing_key:
166
+ specification_version: 4
167
+ summary: Syntax highlight code with Tree-sitter via Kramdown.
168
+ test_files: []