kramdown-syntax_tree_sitter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +295 -0
- data/ext/Rakefile +5 -0
- data/ext/tasks.rake +34 -0
- data/ext/tree_sitter_adapter/Cargo.lock +924 -0
- data/ext/tree_sitter_adapter/Cargo.toml +16 -0
- data/ext/tree_sitter_adapter/src/lib.rs +62 -0
- data/ext/tree_sitter_adapter/src/tree_sitter_adapter.rs +171 -0
- data/lib/kramdown/converter/syntax_highlighter/tree_sitter.rb +35 -0
- data/lib/kramdown/syntax_tree_sitter/version.rb +12 -0
- data/lib/kramdown/syntax_tree_sitter.rb +4 -0
- data/lib/tree_sitter_adapter.rb +8 -0
- metadata +168 -0
@@ -0,0 +1,16 @@
|
|
1
|
+
[package]
|
2
|
+
name = "tree_sitter_adapter"
|
3
|
+
version = "0.0.0"
|
4
|
+
edition = "2021"
|
5
|
+
|
6
|
+
[lib]
|
7
|
+
name = "tree_sitter_adapter"
|
8
|
+
crate-type = ["cdylib"]
|
9
|
+
|
10
|
+
[dependencies]
|
11
|
+
anyhow = "*"
|
12
|
+
tree-sitter = "*"
|
13
|
+
tree-sitter-cli = "*"
|
14
|
+
tree-sitter-highlight = "*"
|
15
|
+
tree-sitter-loader = "*"
|
16
|
+
rutie = "*"
|
@@ -0,0 +1,62 @@
|
|
1
|
+
#[macro_use]
|
2
|
+
extern crate rutie;
|
3
|
+
|
4
|
+
use rutie::{AnyException, Class, Exception, Object, RString, VM};
|
5
|
+
|
6
|
+
mod tree_sitter_adapter;
|
7
|
+
|
8
|
+
class!(TreeSitterAdapter);
|
9
|
+
|
10
|
+
methods!(
|
11
|
+
TreeSitterAdapter,
|
12
|
+
_rtself,
|
13
|
+
fn pub_highlight(raw_code: RString, raw_parsers_dir: RString, raw_scope: RString) -> RString {
|
14
|
+
VM::unwrap_or_raise_ex(
|
15
|
+
tree_sitter_adapter::highlight(
|
16
|
+
&raw_code.unwrap().to_string(),
|
17
|
+
&raw_parsers_dir.unwrap().to_string(),
|
18
|
+
&raw_scope.unwrap().to_string(),
|
19
|
+
)
|
20
|
+
.as_ref()
|
21
|
+
.map(String::as_str)
|
22
|
+
.map(RString::new_utf8)
|
23
|
+
.map_err(String::as_str)
|
24
|
+
.map_err(AnyException::new_runtime_error),
|
25
|
+
)
|
26
|
+
}
|
27
|
+
);
|
28
|
+
|
29
|
+
#[no_mangle]
|
30
|
+
pub extern "C" fn Init_tree_sitter_adapter() {
|
31
|
+
Class::new("TreeSitterAdapter", None).define(|class_| {
|
32
|
+
class_.def_self("highlight", pub_highlight);
|
33
|
+
});
|
34
|
+
}
|
35
|
+
|
36
|
+
pub trait VMExt {
|
37
|
+
fn unwrap_or_raise_ex<T, E>(x: Result<T, E>) -> T
|
38
|
+
where
|
39
|
+
E: Into<AnyException>;
|
40
|
+
}
|
41
|
+
|
42
|
+
impl VMExt for VM {
|
43
|
+
fn unwrap_or_raise_ex<T, E>(result: Result<T, E>) -> T
|
44
|
+
where
|
45
|
+
E: Into<AnyException>,
|
46
|
+
{
|
47
|
+
result.unwrap_or_else(|e| {
|
48
|
+
VM::raise_ex(e);
|
49
|
+
unreachable!();
|
50
|
+
})
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
pub trait AnyExceptionExt {
|
55
|
+
fn new_runtime_error(message: &str) -> Self;
|
56
|
+
}
|
57
|
+
|
58
|
+
impl AnyExceptionExt for AnyException {
|
59
|
+
fn new_runtime_error(message: &str) -> Self {
|
60
|
+
AnyException::new("RuntimeError", Some(message))
|
61
|
+
}
|
62
|
+
}
|
@@ -0,0 +1,171 @@
|
|
1
|
+
use anyhow::{Context, Error, Result};
|
2
|
+
use std::convert;
|
3
|
+
use std::path::PathBuf;
|
4
|
+
use tree_sitter::Language;
|
5
|
+
use tree_sitter_cli::highlight::Theme;
|
6
|
+
use tree_sitter_highlight::{Error as TSError, HighlightEvent};
|
7
|
+
use tree_sitter_highlight::{Highlight, HighlightConfiguration, Highlighter, HtmlRenderer};
|
8
|
+
use tree_sitter_loader::{Config, LanguageConfiguration, Loader};
|
9
|
+
|
10
|
+
const LOADER_ERROR_MSG: &str = "Error loading Tree-sitter parsers from directory";
|
11
|
+
const NO_LANGUAGE_ERROR_MSG: &str = "Error retrieving language configuration for scope";
|
12
|
+
|
13
|
+
pub fn highlight(code: &str, parsers_dir: &str, scope: &str) -> Result<String, String> {
|
14
|
+
let parsers_dir = PathBuf::from(parsers_dir);
|
15
|
+
let theme = Theme::default();
|
16
|
+
let mut loader = Loader::new_from_dir(parsers_dir).map_err(Error::to_formatted_string)?;
|
17
|
+
loader.configure_highlights(&theme.highlight_names);
|
18
|
+
loader
|
19
|
+
.language_configuration_from_scope(scope)
|
20
|
+
.and_then(LanguageConfigurationAdapter::highlight_config)
|
21
|
+
.and_then(|config| {
|
22
|
+
let default_css_style = String::new();
|
23
|
+
let css_attribute_callback = get_css_styles(&theme, &default_css_style);
|
24
|
+
HighlighterAdapter::new(&loader, config)
|
25
|
+
.highlight(code)
|
26
|
+
.and_then(|highlights| render_html(highlights, &css_attribute_callback))
|
27
|
+
})
|
28
|
+
.map_err(Error::to_formatted_string)
|
29
|
+
}
|
30
|
+
|
31
|
+
trait LoaderExt {
|
32
|
+
fn new_from_dir(parser_directory: PathBuf) -> Result<Loader>;
|
33
|
+
|
34
|
+
fn language_configuration_from_scope<'a>(
|
35
|
+
&'a self,
|
36
|
+
scope: &'a str,
|
37
|
+
) -> Result<LanguageConfigurationAdapter<'a>>;
|
38
|
+
}
|
39
|
+
|
40
|
+
impl LoaderExt for Loader {
|
41
|
+
fn new_from_dir(parser_directory: PathBuf) -> Result<Loader> {
|
42
|
+
Loader::new()
|
43
|
+
.and_then(|mut loader| {
|
44
|
+
let config = {
|
45
|
+
let parser_directory = parser_directory.clone();
|
46
|
+
let parser_directories = vec![parser_directory];
|
47
|
+
Config { parser_directories }
|
48
|
+
};
|
49
|
+
loader.find_all_languages(&config)?;
|
50
|
+
Ok(loader)
|
51
|
+
})
|
52
|
+
.with_context(|| {
|
53
|
+
let parser_directory_string = parser_directory.display();
|
54
|
+
format!("{LOADER_ERROR_MSG} '{parser_directory_string}'")
|
55
|
+
})
|
56
|
+
}
|
57
|
+
|
58
|
+
fn language_configuration_from_scope<'a>(
|
59
|
+
&'a self,
|
60
|
+
scope: &'a str,
|
61
|
+
) -> Result<LanguageConfigurationAdapter<'a>> {
|
62
|
+
self.language_configuration_for_scope(scope)
|
63
|
+
.transpose()
|
64
|
+
.context("Language not found")
|
65
|
+
.flatten_()
|
66
|
+
.map(|(language, config)| LanguageConfigurationAdapter { language, config })
|
67
|
+
.with_context(|| format!("{NO_LANGUAGE_ERROR_MSG} '{scope}'"))
|
68
|
+
}
|
69
|
+
}
|
70
|
+
|
71
|
+
struct LanguageConfigurationAdapter<'a> {
|
72
|
+
language: Language,
|
73
|
+
config: &'a LanguageConfiguration<'a>,
|
74
|
+
}
|
75
|
+
|
76
|
+
impl<'a> LanguageConfigurationAdapter<'a> {
|
77
|
+
fn highlight_config(self) -> Result<&'a HighlightConfiguration> {
|
78
|
+
self.config
|
79
|
+
.highlight_config(self.language)
|
80
|
+
.transpose()
|
81
|
+
.context("Another issue")
|
82
|
+
.flatten_()
|
83
|
+
}
|
84
|
+
}
|
85
|
+
|
86
|
+
struct HighlightsAdapter<'a, T: Iterator<Item = Result<HighlightEvent, TSError>>> {
|
87
|
+
code: &'a str,
|
88
|
+
highlights: T,
|
89
|
+
}
|
90
|
+
|
91
|
+
struct HighlighterAdapter<'a> {
|
92
|
+
loader: &'a Loader,
|
93
|
+
config: &'a HighlightConfiguration,
|
94
|
+
highlighter: Highlighter,
|
95
|
+
}
|
96
|
+
|
97
|
+
impl<'a> HighlighterAdapter<'a> {
|
98
|
+
fn new(loader: &'a Loader, config: &'a HighlightConfiguration) -> Self {
|
99
|
+
Self {
|
100
|
+
loader,
|
101
|
+
config,
|
102
|
+
highlighter: Highlighter::new(),
|
103
|
+
}
|
104
|
+
}
|
105
|
+
|
106
|
+
fn highlight(
|
107
|
+
&'a mut self,
|
108
|
+
code: &'a str,
|
109
|
+
) -> Result<HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>>
|
110
|
+
{
|
111
|
+
let Self {
|
112
|
+
loader,
|
113
|
+
config,
|
114
|
+
highlighter,
|
115
|
+
} = self;
|
116
|
+
highlighter
|
117
|
+
.highlight(config, code.as_bytes(), None, |s| {
|
118
|
+
loader.highlight_config_for_injection_string(s)
|
119
|
+
})
|
120
|
+
.map(|highlights| HighlightsAdapter { code, highlights })
|
121
|
+
.map_err(Into::into)
|
122
|
+
}
|
123
|
+
}
|
124
|
+
|
125
|
+
fn render_html<'a, F: Fn(Highlight) -> &'a [u8]>(
|
126
|
+
highlights: HighlightsAdapter<'a, impl Iterator<Item = Result<HighlightEvent, TSError>> + 'a>,
|
127
|
+
css_attribute_callback: &F,
|
128
|
+
) -> Result<String> {
|
129
|
+
let HighlightsAdapter { code, highlights } = highlights;
|
130
|
+
let mut renderer = HtmlRenderer::new();
|
131
|
+
renderer.render(highlights, code.as_bytes(), css_attribute_callback)?;
|
132
|
+
// Remove erroneously appended newline
|
133
|
+
if renderer.html.ends_with(&[b'\n']) && !code.ends_with('\n') {
|
134
|
+
renderer.html.pop();
|
135
|
+
}
|
136
|
+
Ok(renderer.lines().collect())
|
137
|
+
}
|
138
|
+
|
139
|
+
fn get_css_styles<'a>(
|
140
|
+
theme: &'a Theme,
|
141
|
+
default_css_style: &'a String,
|
142
|
+
) -> impl Fn(Highlight) -> &'a [u8] {
|
143
|
+
|highlight| {
|
144
|
+
theme
|
145
|
+
.styles
|
146
|
+
.get(highlight.0)
|
147
|
+
.and_then(|style| style.css.as_ref())
|
148
|
+
.unwrap_or(default_css_style)
|
149
|
+
.as_bytes()
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
trait ResultExt<T, E> {
|
154
|
+
fn flatten_(self) -> Result<T, E>;
|
155
|
+
}
|
156
|
+
|
157
|
+
impl<T, E> ResultExt<T, E> for Result<Result<T, E>, E> {
|
158
|
+
fn flatten_(self) -> Result<T, E> {
|
159
|
+
self.and_then(convert::identity)
|
160
|
+
}
|
161
|
+
}
|
162
|
+
|
163
|
+
trait ErrorExt {
|
164
|
+
fn to_formatted_string(self) -> String;
|
165
|
+
}
|
166
|
+
|
167
|
+
impl ErrorExt for Error {
|
168
|
+
fn to_formatted_string(self) -> String {
|
169
|
+
format!("{self:#}")
|
170
|
+
}
|
171
|
+
}
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'kramdown'
|
4
|
+
require 'tree_sitter_adapter'
|
5
|
+
|
6
|
+
module Kramdown
|
7
|
+
module Converter # rubocop:disable Style/Documentation
|
8
|
+
module SyntaxHighlighter
|
9
|
+
# This highlighter is not yet fully configured to highlight code.
|
10
|
+
#
|
11
|
+
# Currently it merely escapes the code so that it can be safely inserted into HTML
|
12
|
+
# text.
|
13
|
+
module TreeSitter
|
14
|
+
DEFAULT_PARSERS_DIR = '~/tree_sitter_parsers'
|
15
|
+
|
16
|
+
def self.call(converter, raw_text, language, type, _)
|
17
|
+
return nil unless language
|
18
|
+
|
19
|
+
parsers_dir = get_option(converter, :tree_sitter_parsers_dir)
|
20
|
+
.then { _1 || DEFAULT_PARSERS_DIR }
|
21
|
+
.then { File.expand_path _1 }
|
22
|
+
rendered_text = TreeSitterAdapter.highlight raw_text, parsers_dir, language
|
23
|
+
# Code blocks are additionally wrapped in HTML code tags
|
24
|
+
type == :block ? "<pre><code>#{rendered_text}</code></pre>" : rendered_text
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.get_option(converter, name)
|
28
|
+
converter.options[:syntax_highlighter_opts][name]
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
add_syntax_highlighter(:'tree-sitter', SyntaxHighlighter::TreeSitter)
|
34
|
+
end
|
35
|
+
end
|
metadata
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: kramdown-syntax_tree_sitter
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Andrew T. Biehl
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2022-11-11 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: kramdown
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '2.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '2.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '13.0'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '13.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rutie
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.0.4
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.0.4
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: minitest
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '5.0'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '5.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: rouge
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '4.0'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '4.0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: rubocop
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '1.36'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '1.36'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: rubocop-minitest
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0.22'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - "~>"
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0.22'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: rubocop-rake
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0.6'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0.6'
|
125
|
+
description:
|
126
|
+
email:
|
127
|
+
executables: []
|
128
|
+
extensions:
|
129
|
+
- ext/Rakefile
|
130
|
+
extra_rdoc_files: []
|
131
|
+
files:
|
132
|
+
- LICENSE.txt
|
133
|
+
- README.md
|
134
|
+
- ext/Rakefile
|
135
|
+
- ext/tasks.rake
|
136
|
+
- ext/tree_sitter_adapter/Cargo.lock
|
137
|
+
- ext/tree_sitter_adapter/Cargo.toml
|
138
|
+
- ext/tree_sitter_adapter/src/lib.rs
|
139
|
+
- ext/tree_sitter_adapter/src/tree_sitter_adapter.rs
|
140
|
+
- lib/kramdown/converter/syntax_highlighter/tree_sitter.rb
|
141
|
+
- lib/kramdown/syntax_tree_sitter.rb
|
142
|
+
- lib/kramdown/syntax_tree_sitter/version.rb
|
143
|
+
- lib/tree_sitter_adapter.rb
|
144
|
+
homepage: https://github.com/andrewtbiehl/kramdown-syntax_tree_sitter
|
145
|
+
licenses:
|
146
|
+
- MIT
|
147
|
+
metadata:
|
148
|
+
rubygems_mfa_required: 'true'
|
149
|
+
post_install_message:
|
150
|
+
rdoc_options: []
|
151
|
+
require_paths:
|
152
|
+
- lib
|
153
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - ">="
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '2.7'
|
158
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
|
+
requirements:
|
160
|
+
- - ">="
|
161
|
+
- !ruby/object:Gem::Version
|
162
|
+
version: '0'
|
163
|
+
requirements: []
|
164
|
+
rubygems_version: 3.3.7
|
165
|
+
signing_key:
|
166
|
+
specification_version: 4
|
167
|
+
summary: Syntax highlight code with Tree-sitter via Kramdown.
|
168
|
+
test_files: []
|