rouge 3.23.0 → 3.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rouge.rb +1 -0
- data/lib/rouge/cli.rb +32 -2
- data/lib/rouge/demos/email +11 -0
- data/lib/rouge/demos/j +12 -0
- data/lib/rouge/formatters/html_line_highlighter.rb +26 -0
- data/lib/rouge/lexer.rb +38 -20
- data/lib/rouge/lexers/apex.rb +9 -7
- data/lib/rouge/lexers/elm.rb +5 -5
- data/lib/rouge/lexers/email.rb +39 -0
- data/lib/rouge/lexers/ghc_core.rb +2 -1
- data/lib/rouge/lexers/graphql.rb +1 -1
- data/lib/rouge/lexers/http.rb +8 -2
- data/lib/rouge/lexers/isbl.rb +2 -2
- data/lib/rouge/lexers/j.rb +244 -0
- data/lib/rouge/lexers/janet.rb +9 -8
- data/lib/rouge/lexers/javascript.rb +7 -2
- data/lib/rouge/lexers/jsl.rb +1 -1
- data/lib/rouge/lexers/jsonnet.rb +4 -3
- data/lib/rouge/lexers/jsp.rb +2 -3
- data/lib/rouge/lexers/kotlin.rb +4 -3
- data/lib/rouge/lexers/opentype_feature_file.rb +0 -1
- data/lib/rouge/lexers/q.rb +1 -1
- data/lib/rouge/lexers/ruby.rb +2 -2
- data/lib/rouge/lexers/rust.rb +1 -0
- data/lib/rouge/lexers/smarty.rb +1 -1
- data/lib/rouge/lexers/wollok.rb +0 -1
- data/lib/rouge/regex_lexer.rb +56 -1
- data/lib/rouge/version.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ff5fdf4ccf2083c09f6ce1c899d36f7ef50b2f4bb70e5fa04dc18ce239422f8a
|
4
|
+
data.tar.gz: e40152f9290403580be90d096219bf81a92750e8c02be04f05d6929b1fb186ba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e9c0bcf9114cb6b260d611777dbeaa7649bc20ad298ddc73fa2b606ca871615051c7f46abfbed3d8121ec08b505e1c3c4c86c2140d5c30a6c6104dae12d15b11
|
7
|
+
data.tar.gz: 061b7d77b9162d45ff5416fefa9a2052199745b31e3897170c79e46311a6b36de959b1f030a903130dda3941c0ca217c13012f27a5c840f0c07b0912b02a93c7
|
data/lib/rouge.rb
CHANGED
@@ -83,6 +83,7 @@ Rouge.load_file 'formatters/html_table'
|
|
83
83
|
Rouge.load_file 'formatters/html_pygments'
|
84
84
|
Rouge.load_file 'formatters/html_legacy'
|
85
85
|
Rouge.load_file 'formatters/html_linewise'
|
86
|
+
Rouge.load_file 'formatters/html_line_highlighter'
|
86
87
|
Rouge.load_file 'formatters/html_line_table'
|
87
88
|
Rouge.load_file 'formatters/html_inline'
|
88
89
|
Rouge.load_file 'formatters/terminal256'
|
data/lib/rouge/cli.rb
CHANGED
@@ -44,6 +44,7 @@ module Rouge
|
|
44
44
|
yield %||
|
45
45
|
yield %|where <command> is one of:|
|
46
46
|
yield %| highlight #{Highlight.desc}|
|
47
|
+
yield %| debug #{Debug.desc}|
|
47
48
|
yield %| help #{Help.desc}|
|
48
49
|
yield %| style #{Style.desc}|
|
49
50
|
yield %| list #{List.desc}|
|
@@ -104,6 +105,8 @@ module Rouge
|
|
104
105
|
Help
|
105
106
|
when 'highlight', 'hi'
|
106
107
|
Highlight
|
108
|
+
when 'debug'
|
109
|
+
Debug
|
107
110
|
when 'style'
|
108
111
|
Style
|
109
112
|
when 'list'
|
@@ -215,7 +218,7 @@ module Rouge
|
|
215
218
|
end
|
216
219
|
end
|
217
220
|
|
218
|
-
def self.
|
221
|
+
def self.parse_opts(argv)
|
219
222
|
opts = {
|
220
223
|
:formatter => supports_truecolor? ? 'terminal-truecolor' : 'terminal256',
|
221
224
|
:theme => 'thankful_eyes',
|
@@ -256,7 +259,11 @@ module Rouge
|
|
256
259
|
end
|
257
260
|
end
|
258
261
|
|
259
|
-
|
262
|
+
opts
|
263
|
+
end
|
264
|
+
|
265
|
+
def self.parse(argv)
|
266
|
+
new(parse_opts(argv))
|
260
267
|
end
|
261
268
|
|
262
269
|
def input_stream
|
@@ -344,6 +351,29 @@ module Rouge
|
|
344
351
|
end
|
345
352
|
end
|
346
353
|
|
354
|
+
class Debug < Highlight
|
355
|
+
def self.desc
|
356
|
+
end
|
357
|
+
|
358
|
+
def self.doc
|
359
|
+
return enum_for(:doc) unless block_given?
|
360
|
+
|
361
|
+
yield %|usage: rougify debug [<options>]|
|
362
|
+
yield %||
|
363
|
+
yield %|Debug a lexer. Similar options to `rougify highlight`, but|
|
364
|
+
yield %|defaults to the `null` formatter, and ensures the `debug`|
|
365
|
+
yield %|option is enabled, to print debugging information to stdout.|
|
366
|
+
end
|
367
|
+
|
368
|
+
def self.parse_opts(argv)
|
369
|
+
out = super(argv)
|
370
|
+
out[:lexer_opts]['debug'] = '1'
|
371
|
+
out[:formatter] = 'null'
|
372
|
+
|
373
|
+
out
|
374
|
+
end
|
375
|
+
end
|
376
|
+
|
347
377
|
class Style < CLI
|
348
378
|
def self.desc
|
349
379
|
"print CSS styles"
|
@@ -0,0 +1,11 @@
|
|
1
|
+
From: Me <me@example.com>
|
2
|
+
To: You <you@example.com>
|
3
|
+
Date: Tue, 21 Jul 2020 15:14:03 +0000
|
4
|
+
Subject: A very important message
|
5
|
+
|
6
|
+
> Please investigate. Thank you.
|
7
|
+
|
8
|
+
I have investigated.
|
9
|
+
|
10
|
+
--
|
11
|
+
This message is highly confidential and will self-destruct.
|
data/lib/rouge/demos/j
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
NB. Procedural programming
|
2
|
+
fizzbuzz=: monad define
|
3
|
+
for_i. >:i.y do.
|
4
|
+
if. 0 = 15 | i do. echo'FizzBuzz'
|
5
|
+
elseif. 0 = 3 | i do. echo'Fizz'
|
6
|
+
elseif. 0 = 5 | i do. echo'Buzz'
|
7
|
+
else. echo i
|
8
|
+
end.
|
9
|
+
end.
|
10
|
+
)
|
11
|
+
NB. Loopless programming
|
12
|
+
fizzbuzz=: echo@(, ::] ('Fizz' ; 'Buzz') ;@#~ 0 = 3 5&|)@>:@i.
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# -*- coding: utf-8 -*- #
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Rouge
|
5
|
+
module Formatters
|
6
|
+
class HTMLLineHighlighter < Formatter
|
7
|
+
tag 'html_line_highlighter'
|
8
|
+
|
9
|
+
def initialize(delegate, opts = {})
|
10
|
+
@delegate = delegate
|
11
|
+
@highlight_line_class = opts.fetch(:highlight_line_class, 'hll')
|
12
|
+
@highlight_lines = opts[:highlight_lines] || []
|
13
|
+
end
|
14
|
+
|
15
|
+
def stream(tokens)
|
16
|
+
lineno = 0
|
17
|
+
token_lines(tokens) do |tokens_in_line|
|
18
|
+
lineno += 1
|
19
|
+
line = %(#{@delegate.format(tokens_in_line)}\n)
|
20
|
+
line = %(<span class="#{@highlight_line_class}">#{line}</span>) if @highlight_lines.include? lineno
|
21
|
+
yield line
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
data/lib/rouge/lexer.rb
CHANGED
@@ -38,28 +38,15 @@ module Rouge
|
|
38
38
|
registry[name.to_s]
|
39
39
|
end
|
40
40
|
|
41
|
-
#
|
42
|
-
#
|
43
|
-
#
|
44
|
-
#
|
45
|
-
# Lexer.find_fancy('erb?parent=tex')
|
46
|
-
#
|
47
|
-
# * You can pass the special name 'guess' so we guess for you,
|
48
|
-
# and you can pass a second argument of the code to guess by
|
49
|
-
#
|
50
|
-
# Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
|
41
|
+
# Same as ::find_fancy, except instead of returning an instantiated
|
42
|
+
# lexer, returns a pair of [lexer_class, options], so that you can
|
43
|
+
# modify or provide additional options to the lexer.
|
51
44
|
#
|
52
|
-
#
|
53
|
-
|
54
|
-
#
|
55
|
-
# This is used in the Redcarpet plugin as well as Rouge's own
|
56
|
-
# markdown lexer for highlighting internal code blocks.
|
57
|
-
#
|
58
|
-
def find_fancy(str, code=nil, additional_options={})
|
59
|
-
|
45
|
+
# Please note: the lexer class might be nil!
|
46
|
+
def lookup_fancy(str, code=nil, default_options={})
|
60
47
|
if str && !str.include?('?') && str != 'guess'
|
61
48
|
lexer_class = find(str)
|
62
|
-
return lexer_class
|
49
|
+
return [lexer_class, default_options]
|
63
50
|
end
|
64
51
|
|
65
52
|
name, opts = str ? str.split('?', 2) : [nil, '']
|
@@ -75,7 +62,7 @@ module Rouge
|
|
75
62
|
[ k.to_s, val ]
|
76
63
|
end
|
77
64
|
|
78
|
-
opts =
|
65
|
+
opts = default_options.merge(Hash[opts])
|
79
66
|
|
80
67
|
lexer_class = case name
|
81
68
|
when 'guess', nil
|
@@ -84,6 +71,29 @@ module Rouge
|
|
84
71
|
self.find(name)
|
85
72
|
end
|
86
73
|
|
74
|
+
[lexer_class, opts]
|
75
|
+
end
|
76
|
+
|
77
|
+
# Find a lexer, with fancy shiny features.
|
78
|
+
#
|
79
|
+
# * The string you pass can include CGI-style options
|
80
|
+
#
|
81
|
+
# Lexer.find_fancy('erb?parent=tex')
|
82
|
+
#
|
83
|
+
# * You can pass the special name 'guess' so we guess for you,
|
84
|
+
# and you can pass a second argument of the code to guess by
|
85
|
+
#
|
86
|
+
# Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
|
87
|
+
#
|
88
|
+
# If the code matches more than one lexer then Guesser::Ambiguous
|
89
|
+
# is raised.
|
90
|
+
#
|
91
|
+
# This is used in the Redcarpet plugin as well as Rouge's own
|
92
|
+
# markdown lexer for highlighting internal code blocks.
|
93
|
+
#
|
94
|
+
def find_fancy(str, code=nil, default_options={})
|
95
|
+
lexer_class, opts = lookup_fancy(str, code, default_options)
|
96
|
+
|
87
97
|
lexer_class && lexer_class.new(opts)
|
88
98
|
end
|
89
99
|
|
@@ -317,6 +327,14 @@ module Rouge
|
|
317
327
|
@debug = Lexer.debug_enabled? && bool_option('debug')
|
318
328
|
end
|
319
329
|
|
330
|
+
# Returns a new lexer with the given options set. Useful for e.g. setting
|
331
|
+
# debug flags post hoc, or providing global overrides for certain options
|
332
|
+
def with(opts={})
|
333
|
+
new_options = @options.dup
|
334
|
+
opts.each { |k, v| new_options[k.to_s] = v }
|
335
|
+
self.class.new(new_options)
|
336
|
+
end
|
337
|
+
|
320
338
|
def as_bool(val)
|
321
339
|
case val
|
322
340
|
when nil, false, 0, '0', 'false', 'off'
|
data/lib/rouge/lexers/apex.rb
CHANGED
@@ -49,7 +49,7 @@ module Rouge
|
|
49
49
|
|
50
50
|
state :root do
|
51
51
|
rule %r/\s+/m, Text
|
52
|
-
|
52
|
+
|
53
53
|
rule %r(//.*?$), Comment::Single
|
54
54
|
rule %r(/\*.*?\*/)m, Comment::Multiline
|
55
55
|
|
@@ -57,17 +57,19 @@ module Rouge
|
|
57
57
|
rule %r/import\b/, Keyword::Namespace, :import
|
58
58
|
|
59
59
|
rule %r/([@$.]?)(#{id})([:(]?)/io do |m|
|
60
|
-
|
60
|
+
lowercased = m[0].downcase
|
61
|
+
uppercased = m[0].upcase
|
62
|
+
if self.class.keywords.include? lowercased
|
61
63
|
token Keyword
|
62
|
-
elsif self.class.soql.include?
|
64
|
+
elsif self.class.soql.include? uppercased
|
63
65
|
token Keyword
|
64
|
-
elsif self.class.declarations.include?
|
66
|
+
elsif self.class.declarations.include? lowercased
|
65
67
|
token Keyword::Declaration
|
66
|
-
elsif self.class.types.include?
|
68
|
+
elsif self.class.types.include? lowercased
|
67
69
|
token Keyword::Type
|
68
|
-
elsif self.class.constants.include?
|
70
|
+
elsif self.class.constants.include? lowercased
|
69
71
|
token Keyword::Constant
|
70
|
-
elsif 'package'
|
72
|
+
elsif lowercased == 'package'
|
71
73
|
token Keyword::Namespace
|
72
74
|
elsif m[1] == "@"
|
73
75
|
token Name::Decorator
|
data/lib/rouge/lexers/elm.rb
CHANGED
@@ -78,13 +78,13 @@ module Rouge
|
|
78
78
|
rule %r/"/, Str::Double, :pop!
|
79
79
|
end
|
80
80
|
|
81
|
-
# Multiple line string with
|
81
|
+
# Multiple line string with triple double quotes, e.g. """ multi """
|
82
82
|
state :multiline_string do
|
83
|
-
rule %r
|
84
|
-
rule %r
|
85
|
-
rule %r
|
83
|
+
rule %r/\\"/, Str::Escape
|
84
|
+
rule %r/"""/, Str, :pop!
|
85
|
+
rule %r/[^"]+/, Str
|
86
|
+
rule %r/"/, Str
|
86
87
|
end
|
87
|
-
|
88
88
|
end
|
89
89
|
end
|
90
90
|
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# -*- coding: utf-8 -*- #
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Rouge
|
5
|
+
module Lexers
|
6
|
+
class Email < RegexLexer
|
7
|
+
tag 'email'
|
8
|
+
aliases 'eml', 'e-mail'
|
9
|
+
filenames '*.eml'
|
10
|
+
mimetypes 'message/rfc822'
|
11
|
+
|
12
|
+
title "Email"
|
13
|
+
desc "An email message"
|
14
|
+
|
15
|
+
start do
|
16
|
+
push :fields
|
17
|
+
end
|
18
|
+
|
19
|
+
state :fields do
|
20
|
+
rule %r/[:]/, Operator, :field_body
|
21
|
+
rule %r/[^\n\r:]+/, Name::Tag
|
22
|
+
rule %r/[\n\r]/, Name::Tag
|
23
|
+
end
|
24
|
+
|
25
|
+
state :field_body do
|
26
|
+
rule(/(\r?\n){2}/) { token Text; pop!(2) }
|
27
|
+
rule %r/\r?\n(?![ \v\t\f])/, Text, :pop!
|
28
|
+
rule %r/[^\n\r]+/, Name::Attribute
|
29
|
+
rule %r/[\n\r]/, Name::Attribute
|
30
|
+
end
|
31
|
+
|
32
|
+
state :root do
|
33
|
+
rule %r/\n/, Text
|
34
|
+
rule %r/^>.*/, Comment
|
35
|
+
rule %r/.+/, Text
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
data/lib/rouge/lexers/graphql.rb
CHANGED
data/lib/rouge/lexers/http.rb
CHANGED
@@ -8,16 +8,22 @@ module Rouge
|
|
8
8
|
title "HTTP"
|
9
9
|
desc 'http requests and responses'
|
10
10
|
|
11
|
+
option :content, "the language for the content (default: auto-detect)"
|
12
|
+
|
11
13
|
def self.http_methods
|
12
14
|
@http_methods ||= %w(GET POST PUT DELETE HEAD OPTIONS TRACE PATCH)
|
13
15
|
end
|
14
16
|
|
15
17
|
def content_lexer
|
18
|
+
@content_lexer ||= (lexer_option(:content) || guess_content_lexer)
|
19
|
+
end
|
20
|
+
|
21
|
+
def guess_content_lexer
|
16
22
|
return Lexers::PlainText unless @content_type
|
17
23
|
|
18
|
-
|
24
|
+
Lexer.guess_by_mimetype(@content_type)
|
19
25
|
rescue Lexer::AmbiguousGuess
|
20
|
-
|
26
|
+
Lexers::PlainText
|
21
27
|
end
|
22
28
|
|
23
29
|
start { @content_type = 'text/plain' }
|
data/lib/rouge/lexers/isbl.rb
CHANGED
@@ -41,7 +41,7 @@ module Rouge
|
|
41
41
|
|
42
42
|
state :dotted do
|
43
43
|
mixin :whitespace
|
44
|
-
rule %r/[a-zа-яё_0-9]
|
44
|
+
rule %r/[a-zа-яё_0-9]+/i do |m|
|
45
45
|
name = m[0]
|
46
46
|
if self.class.constants.include? name.downcase
|
47
47
|
token Name::Builtin
|
@@ -56,7 +56,7 @@ module Rouge
|
|
56
56
|
|
57
57
|
state :type do
|
58
58
|
mixin :whitespace
|
59
|
-
rule %r/[a-zа-яё_0-9]
|
59
|
+
rule %r/[a-zа-яё_0-9]+/i do |m|
|
60
60
|
name = m[0]
|
61
61
|
if self.class.interfaces.include? name.downcase
|
62
62
|
token Keyword::Type
|
@@ -0,0 +1,244 @@
|
|
1
|
+
# -*- coding: utf-8 -*- #
|
2
|
+
# frozen_string_literal: true
|
3
|
+
|
4
|
+
module Rouge
|
5
|
+
module Lexers
|
6
|
+
class J < RegexLexer
|
7
|
+
title 'J'
|
8
|
+
desc "The J programming language (jsoftware.com)"
|
9
|
+
tag 'j'
|
10
|
+
filenames '*.ijs', '*.ijt'
|
11
|
+
|
12
|
+
# For J-specific terms we use, see:
|
13
|
+
# https://code.jsoftware.com/wiki/Vocabulary/AET
|
14
|
+
# https://code.jsoftware.com/wiki/Vocabulary/Glossary
|
15
|
+
|
16
|
+
# https://code.jsoftware.com/wiki/Vocabulary/PartsOfSpeech
|
17
|
+
def self.token_map
|
18
|
+
@token_map ||= {
|
19
|
+
noun: Keyword::Constant,
|
20
|
+
verb: Name::Function,
|
21
|
+
modifier: Operator,
|
22
|
+
name: Name,
|
23
|
+
param: Name::Builtin::Pseudo,
|
24
|
+
other: Punctuation,
|
25
|
+
nil => Error,
|
26
|
+
}
|
27
|
+
end
|
28
|
+
|
29
|
+
# https://code.jsoftware.com/wiki/NuVoc
|
30
|
+
def self.inflection_list
|
31
|
+
@inflection_list ||= ['', '.', ':', '..', '.:', ':.', '::']
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.primitive_table
|
35
|
+
@primitive_table ||= Hash.new([:name]).tap do |h|
|
36
|
+
{
|
37
|
+
'()' => [:other],
|
38
|
+
'=' => [:verb, :other, :other],
|
39
|
+
'<>+-*%$|,#' => [:verb, :verb, :verb],
|
40
|
+
'^' => [:verb, :verb, :modifier],
|
41
|
+
'~"' => [:modifier, :verb, :verb],
|
42
|
+
'.:@' => [:modifier, :modifier, :modifier],
|
43
|
+
';' => [:verb, :modifier, :verb],
|
44
|
+
'!' => [:verb, :modifier, :modifier],
|
45
|
+
'/\\' => [:modifier, :modifier, :verb],
|
46
|
+
'[' => [:verb, nil, :verb],
|
47
|
+
']' => [:verb],
|
48
|
+
'{' => [:verb, :verb, :verb, nil, nil, nil, :verb],
|
49
|
+
'}' => [:modifier, :verb, :verb, nil, nil, nil, :modifier],
|
50
|
+
'`' => [:modifier, nil, :modifier],
|
51
|
+
'&' => [:modifier, :modifier, :modifier, nil, :modifier],
|
52
|
+
'?' => [:verb, :verb],
|
53
|
+
'a' => [:name, :noun, :noun],
|
54
|
+
'ACeEIjorv' => [:name, :verb],
|
55
|
+
'bdfHMT' => [:name, :modifier],
|
56
|
+
'Dt' => [:name, :modifier, :modifier],
|
57
|
+
'F' => [:name, :modifier, :modifier, :modifier, :modifier,
|
58
|
+
:modifier, :modifier],
|
59
|
+
'iu' => [:name, :verb, :verb],
|
60
|
+
'L' => [:name, :verb, :modifier],
|
61
|
+
'mny' => [:param],
|
62
|
+
'p' => [:name, :verb, :verb, :verb],
|
63
|
+
'qsZ' => [:name, nil, :verb],
|
64
|
+
'S' => [:name, nil, :modifier],
|
65
|
+
'u' => [:param, :verb, :verb],
|
66
|
+
'v' => [:param, :verb],
|
67
|
+
'x' => [:param, nil, :verb],
|
68
|
+
}.each {|k, v| k.each_char {|c| h[c] = v } }
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.primitive(char, inflection)
|
73
|
+
i = inflection_list.index(inflection) or return Error
|
74
|
+
token_map[primitive_table[char][i]]
|
75
|
+
end
|
76
|
+
|
77
|
+
def self.control_words
|
78
|
+
@control_words ||= Set.new %w(
|
79
|
+
assert break case catch catchd catcht continue do else elseif end
|
80
|
+
fcase for if return select throw try while whilst
|
81
|
+
)
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.control_words_id
|
85
|
+
@control_words_id ||= Set.new %w(for goto label)
|
86
|
+
end
|
87
|
+
|
88
|
+
state :expr do
|
89
|
+
rule %r/\s+/, Text
|
90
|
+
|
91
|
+
rule %r'([!-&(-/:-@\[-^`{-~]|[A-Za-z]\b)([.:]*)' do |m|
|
92
|
+
token J.primitive(m[1], m[2])
|
93
|
+
end
|
94
|
+
|
95
|
+
rule %r/(?:\d|_\d?):([.:]*)/ do |m|
|
96
|
+
token m[1].empty? ? J.token_map[:verb] : Error
|
97
|
+
end
|
98
|
+
|
99
|
+
rule %r/[\d_][\w.]*([.:]*)/ do |m|
|
100
|
+
token m[1].empty? ? Num : Error
|
101
|
+
end
|
102
|
+
|
103
|
+
rule %r/'/, Str::Single, :str
|
104
|
+
|
105
|
+
rule %r/NB\.(?![.:]).*/, Comment::Single
|
106
|
+
|
107
|
+
rule %r/([A-Za-z]\w*)([.:]*)/ do |m|
|
108
|
+
if m[2] == '.'
|
109
|
+
word, sep, id = m[1].partition '_'
|
110
|
+
list = if sep.empty?
|
111
|
+
J.control_words
|
112
|
+
elsif not id.empty?
|
113
|
+
J.control_words_id
|
114
|
+
end
|
115
|
+
if list and list.include? word
|
116
|
+
token Keyword, word + sep
|
117
|
+
token((word == 'for' ? Name : Name::Label), id)
|
118
|
+
token Keyword, m[2]
|
119
|
+
else
|
120
|
+
token Error
|
121
|
+
end
|
122
|
+
else
|
123
|
+
token m[2].empty? ? Name : Error
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
state :str do
|
129
|
+
rule %r/''/, Str::Escape
|
130
|
+
rule %r/[^'\n]+/, Str::Single
|
131
|
+
rule %r/'|$/, Str::Single, :pop!
|
132
|
+
end
|
133
|
+
|
134
|
+
start do
|
135
|
+
@note_next = false
|
136
|
+
end
|
137
|
+
|
138
|
+
state :root do
|
139
|
+
rule %r/\n/ do
|
140
|
+
token Text
|
141
|
+
if @note_next
|
142
|
+
push :note
|
143
|
+
@note_next = false
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
# https://code.jsoftware.com/wiki/Vocabulary/com
|
148
|
+
# https://code.jsoftware.com/wiki/Vocabulary/NounExplicitDefinition
|
149
|
+
rule %r/
|
150
|
+
([0-4]|13|adverb|conjunction|dyad|monad|noun|verb)([\ \t]+)
|
151
|
+
(def(?:ine)?\b|:)(?![.:])([\ \t]*)
|
152
|
+
/x do |m|
|
153
|
+
groups Keyword::Pseudo, Text, Keyword::Pseudo, Text
|
154
|
+
@def_body = (m[1] == '0' || m[1] == 'noun') ? :noun : :code
|
155
|
+
if m[3] == 'define'
|
156
|
+
# stack: [:root]
|
157
|
+
# or [:root, ..., :def_next]
|
158
|
+
pop! if stack.size > 1
|
159
|
+
push @def_body
|
160
|
+
push :def_next # [:root, ..., @def_body, :def_next]
|
161
|
+
else
|
162
|
+
push :expl_def
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
rule %r/^([ \t]*)(Note\b(?![.:]))([ \t\r]*)(?!=[.:]|$)/ do
|
167
|
+
groups Text, Name, Text
|
168
|
+
@note_next = true
|
169
|
+
end
|
170
|
+
|
171
|
+
rule %r/[mnuvxy]\b(?![.:])/, Name
|
172
|
+
mixin :expr
|
173
|
+
end
|
174
|
+
|
175
|
+
state :def_next do
|
176
|
+
rule %r/\n/, Text, :pop!
|
177
|
+
mixin :root
|
178
|
+
end
|
179
|
+
|
180
|
+
state :expl_def do
|
181
|
+
rule %r/0\b(?![.:])/ do
|
182
|
+
token Keyword::Pseudo
|
183
|
+
# stack: [:root, :expl_def]
|
184
|
+
# or [:root, ..., :def_next, :expl_def]
|
185
|
+
pop! if stack.size > 2
|
186
|
+
goto @def_body
|
187
|
+
push :def_next # [:root, ..., @def_body, :def_next]
|
188
|
+
end
|
189
|
+
rule %r/'/ do
|
190
|
+
if @def_body == :noun
|
191
|
+
token Str::Single
|
192
|
+
goto :str
|
193
|
+
else
|
194
|
+
token Punctuation
|
195
|
+
goto :q_expr
|
196
|
+
end
|
197
|
+
end
|
198
|
+
rule(//) { pop! }
|
199
|
+
end
|
200
|
+
|
201
|
+
# `q_expr` lexes the content of a string literal which is a part of an
|
202
|
+
# explicit definition.
|
203
|
+
# e.g. dyad def 'x + y'
|
204
|
+
state :q_expr do
|
205
|
+
rule %r/''/, Str::Single, :q_str
|
206
|
+
rule %r/'|$/, Punctuation, :pop!
|
207
|
+
rule %r/NB\.(?![.:])([^'\n]|'')*/, Comment::Single
|
208
|
+
mixin :expr
|
209
|
+
end
|
210
|
+
|
211
|
+
state :q_str do
|
212
|
+
rule %r/''''/, Str::Escape
|
213
|
+
rule %r/[^'\n]+/, Str::Single
|
214
|
+
rule %r/''/, Str::Single, :pop!
|
215
|
+
rule(/'|$/) { token Punctuation; pop! 2 }
|
216
|
+
end
|
217
|
+
|
218
|
+
state :note do
|
219
|
+
mixin :delimiter
|
220
|
+
rule %r/.+\n?/, Comment::Multiline
|
221
|
+
end
|
222
|
+
|
223
|
+
state :noun do
|
224
|
+
mixin :delimiter
|
225
|
+
rule %r/.+\n?/, Str::Heredoc
|
226
|
+
end
|
227
|
+
|
228
|
+
state :code do
|
229
|
+
mixin :delimiter
|
230
|
+
rule %r/^([ \t]*)(:)([ \t\r]*)$/ do
|
231
|
+
groups Text, Punctuation, Text
|
232
|
+
end
|
233
|
+
mixin :expr
|
234
|
+
end
|
235
|
+
|
236
|
+
state :delimiter do
|
237
|
+
rule %r/^([ \t]*)(\))([ \t\r]*$\n?)/ do
|
238
|
+
groups Text, Punctuation, Text
|
239
|
+
pop!
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
end
|
244
|
+
end
|
data/lib/rouge/lexers/janet.rb
CHANGED
@@ -114,7 +114,7 @@ module Rouge
|
|
114
114
|
end
|
115
115
|
end
|
116
116
|
|
117
|
-
punctuation = %r/[_
|
117
|
+
punctuation = %r/[_!$%^&*+=~<>.?\/-]/o
|
118
118
|
symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
|
119
119
|
|
120
120
|
state :root do
|
@@ -143,20 +143,19 @@ module Rouge
|
|
143
143
|
|
144
144
|
rule %r/\(/, Punctuation, :function
|
145
145
|
|
146
|
-
rule %r/(')([
|
146
|
+
rule %r/(')(@?[(\[{])/ do
|
147
147
|
groups Operator, Punctuation
|
148
148
|
push :quote
|
149
149
|
end
|
150
150
|
|
151
|
-
rule %r/(~)([
|
151
|
+
rule %r/(~)(@?[(\[{])/ do
|
152
152
|
groups Operator, Punctuation
|
153
153
|
push :quasiquote
|
154
154
|
end
|
155
155
|
|
156
156
|
rule %r/[\#~,';\|]/, Operator
|
157
157
|
|
158
|
-
rule %r/@?[({\[]/, Punctuation
|
159
|
-
rule %r/[)}\]]/, Punctuation, :pop!
|
158
|
+
rule %r/@?[(){}\[\]]/, Punctuation
|
160
159
|
|
161
160
|
rule symbol, Name
|
162
161
|
end
|
@@ -169,6 +168,8 @@ module Rouge
|
|
169
168
|
end
|
170
169
|
|
171
170
|
state :function do
|
171
|
+
rule %r/[\)]/, Punctuation, :pop!
|
172
|
+
|
172
173
|
rule symbol do |m|
|
173
174
|
case m[0]
|
174
175
|
when "quote"
|
@@ -187,8 +188,9 @@ module Rouge
|
|
187
188
|
end
|
188
189
|
|
189
190
|
state :quote do
|
190
|
-
rule %r/[
|
191
|
-
rule
|
191
|
+
rule %r/[(\[{]/, Punctuation, :push
|
192
|
+
rule %r/[)\]}]/, Punctuation, :pop!
|
193
|
+
rule symbol, Str::Escape
|
192
194
|
mixin :root
|
193
195
|
end
|
194
196
|
|
@@ -199,7 +201,6 @@ module Rouge
|
|
199
201
|
end
|
200
202
|
rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
|
201
203
|
groups Punctuation, Text, Keyword, Text, Punctuation
|
202
|
-
push :root
|
203
204
|
push :function
|
204
205
|
end
|
205
206
|
|
@@ -162,6 +162,10 @@ module Rouge
|
|
162
162
|
push :template_string
|
163
163
|
end
|
164
164
|
|
165
|
+
# special case for the safe navigation operator ?.
|
166
|
+
# so that we don't start detecting a ternary expr
|
167
|
+
rule %r/[?][.]/, Punctuation
|
168
|
+
|
165
169
|
rule %r/[?]/ do
|
166
170
|
token Punctuation
|
167
171
|
push :ternary
|
@@ -265,9 +269,10 @@ module Rouge
|
|
265
269
|
|
266
270
|
# template strings
|
267
271
|
state :template_string do
|
268
|
-
rule %r
|
272
|
+
rule %r/[$]{/, Punctuation, :template_string_expr
|
269
273
|
rule %r/`/, Str::Double, :pop!
|
270
|
-
rule %r
|
274
|
+
rule %r/\\[$`]/, Str::Escape
|
275
|
+
rule %r/[$]/, Str::Double
|
271
276
|
end
|
272
277
|
|
273
278
|
state :template_string_expr do
|
data/lib/rouge/lexers/jsl.rb
CHANGED
data/lib/rouge/lexers/jsonnet.rb
CHANGED
@@ -126,6 +126,7 @@ module Rouge
|
|
126
126
|
|
127
127
|
state :string do
|
128
128
|
rule %r/\\([\\\/bfnrt]|(u[0-9a-fA-F]{4}))/, Str::Escape
|
129
|
+
rule %r/\\./, Str::Escape
|
129
130
|
end
|
130
131
|
|
131
132
|
state :string_double do
|
@@ -137,15 +138,15 @@ module Rouge
|
|
137
138
|
|
138
139
|
state :string_single do
|
139
140
|
mixin :string
|
140
|
-
rule %r/\\'/, Str::Escape
|
141
141
|
rule %r/'/, Str, :pop!
|
142
142
|
rule %r/[^\\']+/, Str
|
143
143
|
end
|
144
144
|
|
145
145
|
state :string_block do
|
146
146
|
mixin :string
|
147
|
-
rule %r
|
148
|
-
rule %r
|
147
|
+
rule %r/[|][|][|]/, Str, :pop!
|
148
|
+
rule %r/[^|\\]+/, Str
|
149
|
+
rule %r/[|]/, Str
|
149
150
|
end
|
150
151
|
end
|
151
152
|
end
|
data/lib/rouge/lexers/jsp.rb
CHANGED
data/lib/rouge/lexers/kotlin.rb
CHANGED
@@ -133,9 +133,10 @@ module Rouge
|
|
133
133
|
end
|
134
134
|
|
135
135
|
state :comment do
|
136
|
-
rule %r'
|
137
|
-
rule %r'
|
138
|
-
rule %r'
|
136
|
+
rule %r'/[*]', Comment::Multiline, :comment
|
137
|
+
rule %r'[*]/', Comment::Multiline, :pop!
|
138
|
+
rule %r'[^/*]+', Comment::Multiline
|
139
|
+
rule %r'[/*]', Comment::Multiline
|
139
140
|
end
|
140
141
|
end
|
141
142
|
end
|
data/lib/rouge/lexers/q.rb
CHANGED
data/lib/rouge/lexers/ruby.rb
CHANGED
@@ -90,7 +90,7 @@ module Rouge
|
|
90
90
|
end
|
91
91
|
|
92
92
|
state :regex_flags do
|
93
|
-
rule %r/[mixounse]
|
93
|
+
rule %r/[mixounse]+/, Str::Regex, :pop!
|
94
94
|
end
|
95
95
|
|
96
96
|
# double-quoted string and symbol
|
@@ -267,7 +267,7 @@ module Rouge
|
|
267
267
|
end
|
268
268
|
|
269
269
|
state :test_heredoc do
|
270
|
-
rule %r/[^#\\\n]
|
270
|
+
rule %r/[^#\\\n]+$/ do |m|
|
271
271
|
tolerant, heredoc_name = @heredoc_queue.first
|
272
272
|
check = tolerant ? m[0].strip : m[0].rstrip
|
273
273
|
|
data/lib/rouge/lexers/rust.rb
CHANGED
@@ -99,6 +99,7 @@ module Rouge
|
|
99
99
|
rule %r/[*\/!@~&+%^<>=\?-]|\.{2,3}/, Operator
|
100
100
|
|
101
101
|
rule %r/([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
|
102
|
+
rule %r/[.]\s*await\b/, Keyword
|
102
103
|
rule %r/[.]\s*#{id}/, Name::Property
|
103
104
|
rule %r/[.]\s*\d+/, Name::Attribute
|
104
105
|
rule %r/(#{id})(::)/m do
|
data/lib/rouge/lexers/smarty.rb
CHANGED
data/lib/rouge/lexers/wollok.rb
CHANGED
data/lib/rouge/regex_lexer.rb
CHANGED
@@ -6,6 +6,37 @@ module Rouge
|
|
6
6
|
# A stateful lexer that uses sets of regular expressions to
|
7
7
|
# tokenize a string. Most lexers are instances of RegexLexer.
|
8
8
|
class RegexLexer < Lexer
|
9
|
+
class InvalidRegex < StandardError
|
10
|
+
def initialize(re)
|
11
|
+
@re = re
|
12
|
+
end
|
13
|
+
|
14
|
+
def to_s
|
15
|
+
"regex #{@re.inspect} matches empty string, but has no predicate!"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class ClosedState < StandardError
|
20
|
+
attr_reader :state
|
21
|
+
def initialize(state)
|
22
|
+
@state = state
|
23
|
+
end
|
24
|
+
|
25
|
+
def rule
|
26
|
+
@state.rules.last
|
27
|
+
end
|
28
|
+
|
29
|
+
def to_s
|
30
|
+
rule = @state.rules.last
|
31
|
+
msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
|
32
|
+
if rule.re.source.include?('*')
|
33
|
+
msg += " Consider replacing * with +."
|
34
|
+
end
|
35
|
+
|
36
|
+
msg
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
9
40
|
# A rule is a tuple of a regular expression to test, and a callback
|
10
41
|
# to perform if the test succeeds.
|
11
42
|
#
|
@@ -42,12 +73,13 @@ module Rouge
|
|
42
73
|
end
|
43
74
|
|
44
75
|
class StateDSL
|
45
|
-
attr_reader :rules
|
76
|
+
attr_reader :rules, :name
|
46
77
|
def initialize(name, &defn)
|
47
78
|
@name = name
|
48
79
|
@defn = defn
|
49
80
|
@rules = []
|
50
81
|
@loaded = false
|
82
|
+
@closed = false
|
51
83
|
end
|
52
84
|
|
53
85
|
def to_state(lexer_class)
|
@@ -95,10 +127,14 @@ module Rouge
|
|
95
127
|
# {RegexLexer#token}, and {RegexLexer#delegate}. The first
|
96
128
|
# argument can be used to access the match groups.
|
97
129
|
def rule(re, tok=nil, next_state=nil, &callback)
|
130
|
+
raise ClosedState.new(self) if @closed
|
131
|
+
|
98
132
|
if tok.nil? && callback.nil?
|
99
133
|
raise "please pass `rule` a token to yield or a callback"
|
100
134
|
end
|
101
135
|
|
136
|
+
matches_empty = re =~ ''
|
137
|
+
|
102
138
|
callback ||= case next_state
|
103
139
|
when :pop!
|
104
140
|
proc do |stream|
|
@@ -123,6 +159,9 @@ module Rouge
|
|
123
159
|
@stack.push(state)
|
124
160
|
end
|
125
161
|
when nil
|
162
|
+
# cannot use an empty-matching regexp with no predicate
|
163
|
+
raise InvalidRegex.new(re) if matches_empty
|
164
|
+
|
126
165
|
proc do |stream|
|
127
166
|
puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
|
128
167
|
@output_stream.call(tok, stream[0])
|
@@ -132,6 +171,22 @@ module Rouge
|
|
132
171
|
end
|
133
172
|
|
134
173
|
rules << Rule.new(re, callback)
|
174
|
+
|
175
|
+
close! if matches_empty && !context_sensitive?(re)
|
176
|
+
end
|
177
|
+
|
178
|
+
def context_sensitive?(re)
|
179
|
+
source = re.source
|
180
|
+
return true if source =~ /[(][?]<?[!=]/
|
181
|
+
|
182
|
+
# anchors count as lookahead/behind
|
183
|
+
return true if source =~ /[$^]/
|
184
|
+
|
185
|
+
false
|
186
|
+
end
|
187
|
+
|
188
|
+
def close!
|
189
|
+
@closed = true
|
135
190
|
end
|
136
191
|
|
137
192
|
# Mix in the rules from another state into this state. The rules
|
data/lib/rouge/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rouge
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.24.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeanine Adkisson
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-13 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
|
14
14
|
email:
|
@@ -73,6 +73,7 @@ files:
|
|
73
73
|
- lib/rouge/demos/eiffel
|
74
74
|
- lib/rouge/demos/elixir
|
75
75
|
- lib/rouge/demos/elm
|
76
|
+
- lib/rouge/demos/email
|
76
77
|
- lib/rouge/demos/epp
|
77
78
|
- lib/rouge/demos/erb
|
78
79
|
- lib/rouge/demos/erlang
|
@@ -109,6 +110,7 @@ files:
|
|
109
110
|
- lib/rouge/demos/irb
|
110
111
|
- lib/rouge/demos/irb_output
|
111
112
|
- lib/rouge/demos/isbl
|
113
|
+
- lib/rouge/demos/j
|
112
114
|
- lib/rouge/demos/janet
|
113
115
|
- lib/rouge/demos/java
|
114
116
|
- lib/rouge/demos/javascript
|
@@ -228,6 +230,7 @@ files:
|
|
228
230
|
- lib/rouge/formatters/html.rb
|
229
231
|
- lib/rouge/formatters/html_inline.rb
|
230
232
|
- lib/rouge/formatters/html_legacy.rb
|
233
|
+
- lib/rouge/formatters/html_line_highlighter.rb
|
231
234
|
- lib/rouge/formatters/html_line_table.rb
|
232
235
|
- lib/rouge/formatters/html_linewise.rb
|
233
236
|
- lib/rouge/formatters/html_pygments.rb
|
@@ -296,6 +299,7 @@ files:
|
|
296
299
|
- lib/rouge/lexers/eiffel.rb
|
297
300
|
- lib/rouge/lexers/elixir.rb
|
298
301
|
- lib/rouge/lexers/elm.rb
|
302
|
+
- lib/rouge/lexers/email.rb
|
299
303
|
- lib/rouge/lexers/epp.rb
|
300
304
|
- lib/rouge/lexers/erb.rb
|
301
305
|
- lib/rouge/lexers/erlang.rb
|
@@ -333,6 +337,7 @@ files:
|
|
333
337
|
- lib/rouge/lexers/irb.rb
|
334
338
|
- lib/rouge/lexers/isbl.rb
|
335
339
|
- lib/rouge/lexers/isbl/builtins.rb
|
340
|
+
- lib/rouge/lexers/j.rb
|
336
341
|
- lib/rouge/lexers/janet.rb
|
337
342
|
- lib/rouge/lexers/java.rb
|
338
343
|
- lib/rouge/lexers/javascript.rb
|