rouge 3.23.0 → 3.24.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: abcc9751ef5e076b3765942693a8f098d83f746aa7527b683bf1b5772f700b48
4
- data.tar.gz: 06074e941ba684cbe5871dfcaeb1dafca415cd228e7223c679847a137d937693
3
+ metadata.gz: ff5fdf4ccf2083c09f6ce1c899d36f7ef50b2f4bb70e5fa04dc18ce239422f8a
4
+ data.tar.gz: e40152f9290403580be90d096219bf81a92750e8c02be04f05d6929b1fb186ba
5
5
  SHA512:
6
- metadata.gz: 2db089d9c952c5d79a93659bcb95f008630a60ea35b756f905a51f6d583dc7ab17470faa1a17f77c1165fa0ecee495c0d890a2bebdcc4aef5f03465620691283
7
- data.tar.gz: 8a161de0a32d5a3be6ae31d4112949bea3ffdbd9bd6408dc6b4a6fa3359d9b34c4b214c383c71f3927c9ed2f2431e1240de9cf0f4f02f9d098ebad1ba367dd66
6
+ metadata.gz: e9c0bcf9114cb6b260d611777dbeaa7649bc20ad298ddc73fa2b606ca871615051c7f46abfbed3d8121ec08b505e1c3c4c86c2140d5c30a6c6104dae12d15b11
7
+ data.tar.gz: 061b7d77b9162d45ff5416fefa9a2052199745b31e3897170c79e46311a6b36de959b1f030a903130dda3941c0ca217c13012f27a5c840f0c07b0912b02a93c7
@@ -83,6 +83,7 @@ Rouge.load_file 'formatters/html_table'
83
83
  Rouge.load_file 'formatters/html_pygments'
84
84
  Rouge.load_file 'formatters/html_legacy'
85
85
  Rouge.load_file 'formatters/html_linewise'
86
+ Rouge.load_file 'formatters/html_line_highlighter'
86
87
  Rouge.load_file 'formatters/html_line_table'
87
88
  Rouge.load_file 'formatters/html_inline'
88
89
  Rouge.load_file 'formatters/terminal256'
@@ -44,6 +44,7 @@ module Rouge
44
44
  yield %||
45
45
  yield %|where <command> is one of:|
46
46
  yield %| highlight #{Highlight.desc}|
47
+ yield %| debug #{Debug.desc}|
47
48
  yield %| help #{Help.desc}|
48
49
  yield %| style #{Style.desc}|
49
50
  yield %| list #{List.desc}|
@@ -104,6 +105,8 @@ module Rouge
104
105
  Help
105
106
  when 'highlight', 'hi'
106
107
  Highlight
108
+ when 'debug'
109
+ Debug
107
110
  when 'style'
108
111
  Style
109
112
  when 'list'
@@ -215,7 +218,7 @@ module Rouge
215
218
  end
216
219
  end
217
220
 
218
- def self.parse(argv)
221
+ def self.parse_opts(argv)
219
222
  opts = {
220
223
  :formatter => supports_truecolor? ? 'terminal-truecolor' : 'terminal256',
221
224
  :theme => 'thankful_eyes',
@@ -256,7 +259,11 @@ module Rouge
256
259
  end
257
260
  end
258
261
 
259
- new(opts)
262
+ opts
263
+ end
264
+
265
+ def self.parse(argv)
266
+ new(parse_opts(argv))
260
267
  end
261
268
 
262
269
  def input_stream
@@ -344,6 +351,29 @@ module Rouge
344
351
  end
345
352
  end
346
353
 
354
+ class Debug < Highlight
355
+ def self.desc
356
+ end
357
+
358
+ def self.doc
359
+ return enum_for(:doc) unless block_given?
360
+
361
+ yield %|usage: rougify debug [<options>]|
362
+ yield %||
363
+ yield %|Debug a lexer. Similar options to `rougify highlight`, but|
364
+ yield %|defaults to the `null` formatter, and ensures the `debug`|
365
+ yield %|option is enabled, to print debugging information to stdout.|
366
+ end
367
+
368
+ def self.parse_opts(argv)
369
+ out = super(argv)
370
+ out[:lexer_opts]['debug'] = '1'
371
+ out[:formatter] = 'null'
372
+
373
+ out
374
+ end
375
+ end
376
+
347
377
  class Style < CLI
348
378
  def self.desc
349
379
  "print CSS styles"
@@ -0,0 +1,11 @@
1
+ From: Me <me@example.com>
2
+ To: You <you@example.com>
3
+ Date: Tue, 21 Jul 2020 15:14:03 +0000
4
+ Subject: A very important message
5
+
6
+ > Please investigate. Thank you.
7
+
8
+ I have investigated.
9
+
10
+ --
11
+ This message is highly confidential and will self-destruct.
@@ -0,0 +1,12 @@
1
+ NB. Procedural programming
2
+ fizzbuzz=: monad define
3
+ for_i. >:i.y do.
4
+ if. 0 = 15 | i do. echo'FizzBuzz'
5
+ elseif. 0 = 3 | i do. echo'Fizz'
6
+ elseif. 0 = 5 | i do. echo'Buzz'
7
+ else. echo i
8
+ end.
9
+ end.
10
+ )
11
+ NB. Loopless programming
12
+ fizzbuzz=: echo@(, ::] ('Fizz' ; 'Buzz') ;@#~ 0 = 3 5&|)@>:@i.
@@ -0,0 +1,26 @@
1
+ # -*- coding: utf-8 -*- #
2
+ # frozen_string_literal: true
3
+
4
+ module Rouge
5
+ module Formatters
6
+ class HTMLLineHighlighter < Formatter
7
+ tag 'html_line_highlighter'
8
+
9
+ def initialize(delegate, opts = {})
10
+ @delegate = delegate
11
+ @highlight_line_class = opts.fetch(:highlight_line_class, 'hll')
12
+ @highlight_lines = opts[:highlight_lines] || []
13
+ end
14
+
15
+ def stream(tokens)
16
+ lineno = 0
17
+ token_lines(tokens) do |tokens_in_line|
18
+ lineno += 1
19
+ line = %(#{@delegate.format(tokens_in_line)}\n)
20
+ line = %(<span class="#{@highlight_line_class}">#{line}</span>) if @highlight_lines.include? lineno
21
+ yield line
22
+ end
23
+ end
24
+ end
25
+ end
26
+ end
@@ -38,28 +38,15 @@ module Rouge
38
38
  registry[name.to_s]
39
39
  end
40
40
 
41
- # Find a lexer, with fancy shiny features.
42
- #
43
- # * The string you pass can include CGI-style options
44
- #
45
- # Lexer.find_fancy('erb?parent=tex')
46
- #
47
- # * You can pass the special name 'guess' so we guess for you,
48
- # and you can pass a second argument of the code to guess by
49
- #
50
- # Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
41
+ # Same as ::find_fancy, except instead of returning an instantiated
42
+ # lexer, returns a pair of [lexer_class, options], so that you can
43
+ # modify or provide additional options to the lexer.
51
44
  #
52
- # If the code matches more than one lexer then Guesser::Ambiguous
53
- # is raised.
54
- #
55
- # This is used in the Redcarpet plugin as well as Rouge's own
56
- # markdown lexer for highlighting internal code blocks.
57
- #
58
- def find_fancy(str, code=nil, additional_options={})
59
-
45
+ # Please note: the lexer class might be nil!
46
+ def lookup_fancy(str, code=nil, default_options={})
60
47
  if str && !str.include?('?') && str != 'guess'
61
48
  lexer_class = find(str)
62
- return lexer_class && lexer_class.new(additional_options)
49
+ return [lexer_class, default_options]
63
50
  end
64
51
 
65
52
  name, opts = str ? str.split('?', 2) : [nil, '']
@@ -75,7 +62,7 @@ module Rouge
75
62
  [ k.to_s, val ]
76
63
  end
77
64
 
78
- opts = additional_options.merge(Hash[opts])
65
+ opts = default_options.merge(Hash[opts])
79
66
 
80
67
  lexer_class = case name
81
68
  when 'guess', nil
@@ -84,6 +71,29 @@ module Rouge
84
71
  self.find(name)
85
72
  end
86
73
 
74
+ [lexer_class, opts]
75
+ end
76
+
77
+ # Find a lexer, with fancy shiny features.
78
+ #
79
+ # * The string you pass can include CGI-style options
80
+ #
81
+ # Lexer.find_fancy('erb?parent=tex')
82
+ #
83
+ # * You can pass the special name 'guess' so we guess for you,
84
+ # and you can pass a second argument of the code to guess by
85
+ #
86
+ # Lexer.find_fancy('guess', "#!/bin/bash\necho Hello, world")
87
+ #
88
+ # If the code matches more than one lexer then Guesser::Ambiguous
89
+ # is raised.
90
+ #
91
+ # This is used in the Redcarpet plugin as well as Rouge's own
92
+ # markdown lexer for highlighting internal code blocks.
93
+ #
94
+ def find_fancy(str, code=nil, default_options={})
95
+ lexer_class, opts = lookup_fancy(str, code, default_options)
96
+
87
97
  lexer_class && lexer_class.new(opts)
88
98
  end
89
99
 
@@ -317,6 +327,14 @@ module Rouge
317
327
  @debug = Lexer.debug_enabled? && bool_option('debug')
318
328
  end
319
329
 
330
+ # Returns a new lexer with the given options set. Useful for e.g. setting
331
+ # debug flags post hoc, or providing global overrides for certain options
332
+ def with(opts={})
333
+ new_options = @options.dup
334
+ opts.each { |k, v| new_options[k.to_s] = v }
335
+ self.class.new(new_options)
336
+ end
337
+
320
338
  def as_bool(val)
321
339
  case val
322
340
  when nil, false, 0, '0', 'false', 'off'
@@ -49,7 +49,7 @@ module Rouge
49
49
 
50
50
  state :root do
51
51
  rule %r/\s+/m, Text
52
-
52
+
53
53
  rule %r(//.*?$), Comment::Single
54
54
  rule %r(/\*.*?\*/)m, Comment::Multiline
55
55
 
@@ -57,17 +57,19 @@ module Rouge
57
57
  rule %r/import\b/, Keyword::Namespace, :import
58
58
 
59
59
  rule %r/([@$.]?)(#{id})([:(]?)/io do |m|
60
- if self.class.keywords.include? m[0].downcase
60
+ lowercased = m[0].downcase
61
+ uppercased = m[0].upcase
62
+ if self.class.keywords.include? lowercased
61
63
  token Keyword
62
- elsif self.class.soql.include? m[0].upcase
64
+ elsif self.class.soql.include? uppercased
63
65
  token Keyword
64
- elsif self.class.declarations.include? m[0].downcase
66
+ elsif self.class.declarations.include? lowercased
65
67
  token Keyword::Declaration
66
- elsif self.class.types.include? m[0].downcase
68
+ elsif self.class.types.include? lowercased
67
69
  token Keyword::Type
68
- elsif self.class.constants.include? m[0].downcase
70
+ elsif self.class.constants.include? lowercased
69
71
  token Keyword::Constant
70
- elsif 'package'.casecmp m[0]
72
+ elsif lowercased == 'package'
71
73
  token Keyword::Namespace
72
74
  elsif m[1] == "@"
73
75
  token Name::Decorator
@@ -78,13 +78,13 @@ module Rouge
78
78
  rule %r/"/, Str::Double, :pop!
79
79
  end
80
80
 
81
- # Multiple line string with tripple double quotes, e.g. """ multi """
81
+ # Multiple line string with triple double quotes, e.g. """ multi """
82
82
  state :multiline_string do
83
- rule %r/\s*"""/, Str, :pop!
84
- rule %r/.*/, Str
85
- rule %r/\s*/, Str
83
+ rule %r/\\"/, Str::Escape
84
+ rule %r/"""/, Str, :pop!
85
+ rule %r/[^"]+/, Str
86
+ rule %r/"/, Str
86
87
  end
87
-
88
88
  end
89
89
  end
90
90
  end
@@ -0,0 +1,39 @@
1
+ # -*- coding: utf-8 -*- #
2
+ # frozen_string_literal: true
3
+
4
+ module Rouge
5
+ module Lexers
6
+ class Email < RegexLexer
7
+ tag 'email'
8
+ aliases 'eml', 'e-mail'
9
+ filenames '*.eml'
10
+ mimetypes 'message/rfc822'
11
+
12
+ title "Email"
13
+ desc "An email message"
14
+
15
+ start do
16
+ push :fields
17
+ end
18
+
19
+ state :fields do
20
+ rule %r/[:]/, Operator, :field_body
21
+ rule %r/[^\n\r:]+/, Name::Tag
22
+ rule %r/[\n\r]/, Name::Tag
23
+ end
24
+
25
+ state :field_body do
26
+ rule(/(\r?\n){2}/) { token Text; pop!(2) }
27
+ rule %r/\r?\n(?![ \v\t\f])/, Text, :pop!
28
+ rule %r/[^\n\r]+/, Name::Attribute
29
+ rule %r/[\n\r]/, Name::Attribute
30
+ end
31
+
32
+ state :root do
33
+ rule %r/\n/, Text
34
+ rule %r/^>.*/, Comment
35
+ rule %r/.+/, Text
36
+ end
37
+ end
38
+ end
39
+ end
@@ -27,8 +27,9 @@ module Rouge
27
27
  mixin :function
28
28
 
29
29
  # rest is Text
30
+ # TODO: this is really inefficient
30
31
  rule %r/\s/m, Text
31
- rule %r/.*/, Text
32
+ rule %r/./, Text
32
33
  end
33
34
 
34
35
  state :expression do
@@ -20,7 +20,7 @@ module Rouge
20
20
  end
21
21
 
22
22
  rule %r/\bfragment\b/, Keyword, :fragment_definition
23
-
23
+
24
24
  rule %r/\bscalar\b/, Keyword, :value
25
25
 
26
26
  rule %r/\b(?:type|interface|enum)\b/, Keyword, :type_definition
@@ -8,16 +8,22 @@ module Rouge
8
8
  title "HTTP"
9
9
  desc 'http requests and responses'
10
10
 
11
+ option :content, "the language for the content (default: auto-detect)"
12
+
11
13
  def self.http_methods
12
14
  @http_methods ||= %w(GET POST PUT DELETE HEAD OPTIONS TRACE PATCH)
13
15
  end
14
16
 
15
17
  def content_lexer
18
+ @content_lexer ||= (lexer_option(:content) || guess_content_lexer)
19
+ end
20
+
21
+ def guess_content_lexer
16
22
  return Lexers::PlainText unless @content_type
17
23
 
18
- @content_lexer ||= Lexer.guess_by_mimetype(@content_type)
24
+ Lexer.guess_by_mimetype(@content_type)
19
25
  rescue Lexer::AmbiguousGuess
20
- @content_lexer = Lexers::PlainText
26
+ Lexers::PlainText
21
27
  end
22
28
 
23
29
  start { @content_type = 'text/plain' }
@@ -41,7 +41,7 @@ module Rouge
41
41
 
42
42
  state :dotted do
43
43
  mixin :whitespace
44
- rule %r/[a-zа-яё_0-9]*/i do |m|
44
+ rule %r/[a-zа-яё_0-9]+/i do |m|
45
45
  name = m[0]
46
46
  if self.class.constants.include? name.downcase
47
47
  token Name::Builtin
@@ -56,7 +56,7 @@ module Rouge
56
56
 
57
57
  state :type do
58
58
  mixin :whitespace
59
- rule %r/[a-zа-яё_0-9]*/i do |m|
59
+ rule %r/[a-zа-яё_0-9]+/i do |m|
60
60
  name = m[0]
61
61
  if self.class.interfaces.include? name.downcase
62
62
  token Keyword::Type
@@ -0,0 +1,244 @@
1
+ # -*- coding: utf-8 -*- #
2
+ # frozen_string_literal: true
3
+
4
+ module Rouge
5
+ module Lexers
6
+ class J < RegexLexer
7
+ title 'J'
8
+ desc "The J programming language (jsoftware.com)"
9
+ tag 'j'
10
+ filenames '*.ijs', '*.ijt'
11
+
12
+ # For J-specific terms we use, see:
13
+ # https://code.jsoftware.com/wiki/Vocabulary/AET
14
+ # https://code.jsoftware.com/wiki/Vocabulary/Glossary
15
+
16
+ # https://code.jsoftware.com/wiki/Vocabulary/PartsOfSpeech
17
+ def self.token_map
18
+ @token_map ||= {
19
+ noun: Keyword::Constant,
20
+ verb: Name::Function,
21
+ modifier: Operator,
22
+ name: Name,
23
+ param: Name::Builtin::Pseudo,
24
+ other: Punctuation,
25
+ nil => Error,
26
+ }
27
+ end
28
+
29
+ # https://code.jsoftware.com/wiki/NuVoc
30
+ def self.inflection_list
31
+ @inflection_list ||= ['', '.', ':', '..', '.:', ':.', '::']
32
+ end
33
+
34
+ def self.primitive_table
35
+ @primitive_table ||= Hash.new([:name]).tap do |h|
36
+ {
37
+ '()' => [:other],
38
+ '=' => [:verb, :other, :other],
39
+ '<>+-*%$|,#' => [:verb, :verb, :verb],
40
+ '^' => [:verb, :verb, :modifier],
41
+ '~"' => [:modifier, :verb, :verb],
42
+ '.:@' => [:modifier, :modifier, :modifier],
43
+ ';' => [:verb, :modifier, :verb],
44
+ '!' => [:verb, :modifier, :modifier],
45
+ '/\\' => [:modifier, :modifier, :verb],
46
+ '[' => [:verb, nil, :verb],
47
+ ']' => [:verb],
48
+ '{' => [:verb, :verb, :verb, nil, nil, nil, :verb],
49
+ '}' => [:modifier, :verb, :verb, nil, nil, nil, :modifier],
50
+ '`' => [:modifier, nil, :modifier],
51
+ '&' => [:modifier, :modifier, :modifier, nil, :modifier],
52
+ '?' => [:verb, :verb],
53
+ 'a' => [:name, :noun, :noun],
54
+ 'ACeEIjorv' => [:name, :verb],
55
+ 'bdfHMT' => [:name, :modifier],
56
+ 'Dt' => [:name, :modifier, :modifier],
57
+ 'F' => [:name, :modifier, :modifier, :modifier, :modifier,
58
+ :modifier, :modifier],
59
+ 'iu' => [:name, :verb, :verb],
60
+ 'L' => [:name, :verb, :modifier],
61
+ 'mny' => [:param],
62
+ 'p' => [:name, :verb, :verb, :verb],
63
+ 'qsZ' => [:name, nil, :verb],
64
+ 'S' => [:name, nil, :modifier],
65
+ 'u' => [:param, :verb, :verb],
66
+ 'v' => [:param, :verb],
67
+ 'x' => [:param, nil, :verb],
68
+ }.each {|k, v| k.each_char {|c| h[c] = v } }
69
+ end
70
+ end
71
+
72
+ def self.primitive(char, inflection)
73
+ i = inflection_list.index(inflection) or return Error
74
+ token_map[primitive_table[char][i]]
75
+ end
76
+
77
+ def self.control_words
78
+ @control_words ||= Set.new %w(
79
+ assert break case catch catchd catcht continue do else elseif end
80
+ fcase for if return select throw try while whilst
81
+ )
82
+ end
83
+
84
+ def self.control_words_id
85
+ @control_words_id ||= Set.new %w(for goto label)
86
+ end
87
+
88
+ state :expr do
89
+ rule %r/\s+/, Text
90
+
91
+ rule %r'([!-&(-/:-@\[-^`{-~]|[A-Za-z]\b)([.:]*)' do |m|
92
+ token J.primitive(m[1], m[2])
93
+ end
94
+
95
+ rule %r/(?:\d|_\d?):([.:]*)/ do |m|
96
+ token m[1].empty? ? J.token_map[:verb] : Error
97
+ end
98
+
99
+ rule %r/[\d_][\w.]*([.:]*)/ do |m|
100
+ token m[1].empty? ? Num : Error
101
+ end
102
+
103
+ rule %r/'/, Str::Single, :str
104
+
105
+ rule %r/NB\.(?![.:]).*/, Comment::Single
106
+
107
+ rule %r/([A-Za-z]\w*)([.:]*)/ do |m|
108
+ if m[2] == '.'
109
+ word, sep, id = m[1].partition '_'
110
+ list = if sep.empty?
111
+ J.control_words
112
+ elsif not id.empty?
113
+ J.control_words_id
114
+ end
115
+ if list and list.include? word
116
+ token Keyword, word + sep
117
+ token((word == 'for' ? Name : Name::Label), id)
118
+ token Keyword, m[2]
119
+ else
120
+ token Error
121
+ end
122
+ else
123
+ token m[2].empty? ? Name : Error
124
+ end
125
+ end
126
+ end
127
+
128
+ state :str do
129
+ rule %r/''/, Str::Escape
130
+ rule %r/[^'\n]+/, Str::Single
131
+ rule %r/'|$/, Str::Single, :pop!
132
+ end
133
+
134
+ start do
135
+ @note_next = false
136
+ end
137
+
138
+ state :root do
139
+ rule %r/\n/ do
140
+ token Text
141
+ if @note_next
142
+ push :note
143
+ @note_next = false
144
+ end
145
+ end
146
+
147
+ # https://code.jsoftware.com/wiki/Vocabulary/com
148
+ # https://code.jsoftware.com/wiki/Vocabulary/NounExplicitDefinition
149
+ rule %r/
150
+ ([0-4]|13|adverb|conjunction|dyad|monad|noun|verb)([\ \t]+)
151
+ (def(?:ine)?\b|:)(?![.:])([\ \t]*)
152
+ /x do |m|
153
+ groups Keyword::Pseudo, Text, Keyword::Pseudo, Text
154
+ @def_body = (m[1] == '0' || m[1] == 'noun') ? :noun : :code
155
+ if m[3] == 'define'
156
+ # stack: [:root]
157
+ # or [:root, ..., :def_next]
158
+ pop! if stack.size > 1
159
+ push @def_body
160
+ push :def_next # [:root, ..., @def_body, :def_next]
161
+ else
162
+ push :expl_def
163
+ end
164
+ end
165
+
166
+ rule %r/^([ \t]*)(Note\b(?![.:]))([ \t\r]*)(?!=[.:]|$)/ do
167
+ groups Text, Name, Text
168
+ @note_next = true
169
+ end
170
+
171
+ rule %r/[mnuvxy]\b(?![.:])/, Name
172
+ mixin :expr
173
+ end
174
+
175
+ state :def_next do
176
+ rule %r/\n/, Text, :pop!
177
+ mixin :root
178
+ end
179
+
180
+ state :expl_def do
181
+ rule %r/0\b(?![.:])/ do
182
+ token Keyword::Pseudo
183
+ # stack: [:root, :expl_def]
184
+ # or [:root, ..., :def_next, :expl_def]
185
+ pop! if stack.size > 2
186
+ goto @def_body
187
+ push :def_next # [:root, ..., @def_body, :def_next]
188
+ end
189
+ rule %r/'/ do
190
+ if @def_body == :noun
191
+ token Str::Single
192
+ goto :str
193
+ else
194
+ token Punctuation
195
+ goto :q_expr
196
+ end
197
+ end
198
+ rule(//) { pop! }
199
+ end
200
+
201
+ # `q_expr` lexes the content of a string literal which is a part of an
202
+ # explicit definition.
203
+ # e.g. dyad def 'x + y'
204
+ state :q_expr do
205
+ rule %r/''/, Str::Single, :q_str
206
+ rule %r/'|$/, Punctuation, :pop!
207
+ rule %r/NB\.(?![.:])([^'\n]|'')*/, Comment::Single
208
+ mixin :expr
209
+ end
210
+
211
+ state :q_str do
212
+ rule %r/''''/, Str::Escape
213
+ rule %r/[^'\n]+/, Str::Single
214
+ rule %r/''/, Str::Single, :pop!
215
+ rule(/'|$/) { token Punctuation; pop! 2 }
216
+ end
217
+
218
+ state :note do
219
+ mixin :delimiter
220
+ rule %r/.+\n?/, Comment::Multiline
221
+ end
222
+
223
+ state :noun do
224
+ mixin :delimiter
225
+ rule %r/.+\n?/, Str::Heredoc
226
+ end
227
+
228
+ state :code do
229
+ mixin :delimiter
230
+ rule %r/^([ \t]*)(:)([ \t\r]*)$/ do
231
+ groups Text, Punctuation, Text
232
+ end
233
+ mixin :expr
234
+ end
235
+
236
+ state :delimiter do
237
+ rule %r/^([ \t]*)(\))([ \t\r]*$\n?)/ do
238
+ groups Text, Punctuation, Text
239
+ pop!
240
+ end
241
+ end
242
+ end
243
+ end
244
+ end
@@ -114,7 +114,7 @@ module Rouge
114
114
  end
115
115
  end
116
116
 
117
- punctuation = %r/[_!@$%^&*+=~<>.?\/-]/o
117
+ punctuation = %r/[_!$%^&*+=~<>.?\/-]/o
118
118
  symbol = %r/([[:alpha:]]|#{punctuation})([[:word:]]|#{punctuation}|:)*/o
119
119
 
120
120
  state :root do
@@ -143,20 +143,19 @@ module Rouge
143
143
 
144
144
  rule %r/\(/, Punctuation, :function
145
145
 
146
- rule %r/(')([\(\[])/ do
146
+ rule %r/(')(@?[(\[{])/ do
147
147
  groups Operator, Punctuation
148
148
  push :quote
149
149
  end
150
150
 
151
- rule %r/(~)([\(\[])/ do
151
+ rule %r/(~)(@?[(\[{])/ do
152
152
  groups Operator, Punctuation
153
153
  push :quasiquote
154
154
  end
155
155
 
156
156
  rule %r/[\#~,';\|]/, Operator
157
157
 
158
- rule %r/@?[({\[]/, Punctuation, :push
159
- rule %r/[)}\]]/, Punctuation, :pop!
158
+ rule %r/@?[(){}\[\]]/, Punctuation
160
159
 
161
160
  rule symbol, Name
162
161
  end
@@ -169,6 +168,8 @@ module Rouge
169
168
  end
170
169
 
171
170
  state :function do
171
+ rule %r/[\)]/, Punctuation, :pop!
172
+
172
173
  rule symbol do |m|
173
174
  case m[0]
174
175
  when "quote"
@@ -187,8 +188,9 @@ module Rouge
187
188
  end
188
189
 
189
190
  state :quote do
190
- rule %r/[\(\[]/, Punctuation, :push
191
- rule symbol, Str::Symbol
191
+ rule %r/[(\[{]/, Punctuation, :push
192
+ rule %r/[)\]}]/, Punctuation, :pop!
193
+ rule symbol, Str::Escape
192
194
  mixin :root
193
195
  end
194
196
 
@@ -199,7 +201,6 @@ module Rouge
199
201
  end
200
202
  rule %r/(\()(\s*)(unquote)(\s+)(\()/ do
201
203
  groups Punctuation, Text, Keyword, Text, Punctuation
202
- push :root
203
204
  push :function
204
205
  end
205
206
 
@@ -162,6 +162,10 @@ module Rouge
162
162
  push :template_string
163
163
  end
164
164
 
165
+ # special case for the safe navigation operator ?.
166
+ # so that we don't start detecting a ternary expr
167
+ rule %r/[?][.]/, Punctuation
168
+
165
169
  rule %r/[?]/ do
166
170
  token Punctuation
167
171
  push :ternary
@@ -265,9 +269,10 @@ module Rouge
265
269
 
266
270
  # template strings
267
271
  state :template_string do
268
- rule %r/\${/, Punctuation, :template_string_expr
272
+ rule %r/[$]{/, Punctuation, :template_string_expr
269
273
  rule %r/`/, Str::Double, :pop!
270
- rule %r/(\\\\|\\[\$`]|[^\$`]|\$(?!{))*/, Str::Double
274
+ rule %r/\\[$`]/, Str::Escape
275
+ rule %r/[$]/, Str::Double
271
276
  end
272
277
 
273
278
  state :template_string_expr do
@@ -48,7 +48,7 @@ module Rouge
48
48
  rule %r/\\![btrnNf0\\"]/, Str::Escape
49
49
  rule %r/\\/, Str::Double
50
50
  rule %r/"/, Str::Double, :pop!
51
- rule %r/[^\\"]*/m, Str::Double
51
+ rule %r/[^\\"]+/m, Str::Double
52
52
  end
53
53
  end
54
54
  end
@@ -126,6 +126,7 @@ module Rouge
126
126
 
127
127
  state :string do
128
128
  rule %r/\\([\\\/bfnrt]|(u[0-9a-fA-F]{4}))/, Str::Escape
129
+ rule %r/\\./, Str::Escape
129
130
  end
130
131
 
131
132
  state :string_double do
@@ -137,15 +138,15 @@ module Rouge
137
138
 
138
139
  state :string_single do
139
140
  mixin :string
140
- rule %r/\\'/, Str::Escape
141
141
  rule %r/'/, Str, :pop!
142
142
  rule %r/[^\\']+/, Str
143
143
  end
144
144
 
145
145
  state :string_block do
146
146
  mixin :string
147
- rule %r/\|\|\|/, Str, :pop!
148
- rule %r/.*/, Str
147
+ rule %r/[|][|][|]/, Str, :pop!
148
+ rule %r/[^|\\]+/, Str
149
+ rule %r/[|]/, Str
149
150
  end
150
151
  end
151
152
  end
@@ -112,9 +112,8 @@ module Rouge
112
112
 
113
113
  state :jsp_interp_literal_start do
114
114
  rule %r/'/, Literal, :pop!
115
- rule %r/[^']*/, Literal
115
+ rule %r/[^']+/, Literal
116
116
  end
117
-
118
117
  end
119
118
  end
120
- end
119
+ end
@@ -133,9 +133,10 @@ module Rouge
133
133
  end
134
134
 
135
135
  state :comment do
136
- rule %r'\s*/[*].*', Comment::Multiline, :comment
137
- rule %r'.*[*]/', Comment::Multiline, :pop!
138
- rule %r'.*', Comment::Multiline
136
+ rule %r'/[*]', Comment::Multiline, :comment
137
+ rule %r'[*]/', Comment::Multiline, :pop!
138
+ rule %r'[^/*]+', Comment::Multiline
139
+ rule %r'[/*]', Comment::Multiline
139
140
  end
140
141
  end
141
142
  end
@@ -91,7 +91,6 @@ module Rouge
91
91
  state :strings do
92
92
  rule %r/"/, Str, :pop!
93
93
  rule %r/[^"%\n]+/, Str
94
- rule %r/(\([a-z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?/i, Str
95
94
  end
96
95
  end
97
96
  end
@@ -118,7 +118,7 @@ module Rouge
118
118
  end
119
119
 
120
120
  state :bottom do
121
- rule %r/.*\z/m, Comment::Multiline
121
+ rule %r/.+\z/m, Comment::Multiline
122
122
  end
123
123
  end
124
124
  end
@@ -90,7 +90,7 @@ module Rouge
90
90
  end
91
91
 
92
92
  state :regex_flags do
93
- rule %r/[mixounse]*/, Str::Regex, :pop!
93
+ rule %r/[mixounse]+/, Str::Regex, :pop!
94
94
  end
95
95
 
96
96
  # double-quoted string and symbol
@@ -267,7 +267,7 @@ module Rouge
267
267
  end
268
268
 
269
269
  state :test_heredoc do
270
- rule %r/[^#\\\n]*$/ do |m|
270
+ rule %r/[^#\\\n]+$/ do |m|
271
271
  tolerant, heredoc_name = @heredoc_queue.first
272
272
  check = tolerant ? m[0].strip : m[0].rstrip
273
273
 
@@ -99,6 +99,7 @@ module Rouge
99
99
  rule %r/[*\/!@~&+%^<>=\?-]|\.{2,3}/, Operator
100
100
 
101
101
  rule %r/([.]\s*)?#{id}(?=\s*[(])/m, Name::Function
102
+ rule %r/[.]\s*await\b/, Keyword
102
103
  rule %r/[.]\s*#{id}/, Name::Property
103
104
  rule %r/[.]\s*\d+/, Name::Attribute
104
105
  rule %r/(#{id})(::)/m do
@@ -41,7 +41,7 @@ module Rouge
41
41
  end
42
42
 
43
43
 
44
- rule(/.*?(?={[\/a-zA-Z0-9$#*"'])|.*/m) { delegate parent }
44
+ rule(/.+?(?={[\/a-zA-Z0-9$#*"'])/m) { delegate parent }
45
45
  rule(/.+/m) { delegate parent }
46
46
  end
47
47
 
@@ -18,7 +18,6 @@ module Rouge
18
18
 
19
19
  state :whitespaces_and_comments do
20
20
  rule %r/\s+/m, Text::Whitespace
21
- rule %r/$+/m, Text::Whitespace
22
21
  rule %r(//.*$), Comment::Single
23
22
  rule %r(/\*(.|\s)*?\*/)m, Comment::Multiline
24
23
  end
@@ -6,6 +6,37 @@ module Rouge
6
6
  # A stateful lexer that uses sets of regular expressions to
7
7
  # tokenize a string. Most lexers are instances of RegexLexer.
8
8
  class RegexLexer < Lexer
9
+ class InvalidRegex < StandardError
10
+ def initialize(re)
11
+ @re = re
12
+ end
13
+
14
+ def to_s
15
+ "regex #{@re.inspect} matches empty string, but has no predicate!"
16
+ end
17
+ end
18
+
19
+ class ClosedState < StandardError
20
+ attr_reader :state
21
+ def initialize(state)
22
+ @state = state
23
+ end
24
+
25
+ def rule
26
+ @state.rules.last
27
+ end
28
+
29
+ def to_s
30
+ rule = @state.rules.last
31
+ msg = "State :#{state.name} cannot continue after #{rule.inspect}, which will always match."
32
+ if rule.re.source.include?('*')
33
+ msg += " Consider replacing * with +."
34
+ end
35
+
36
+ msg
37
+ end
38
+ end
39
+
9
40
  # A rule is a tuple of a regular expression to test, and a callback
10
41
  # to perform if the test succeeds.
11
42
  #
@@ -42,12 +73,13 @@ module Rouge
42
73
  end
43
74
 
44
75
  class StateDSL
45
- attr_reader :rules
76
+ attr_reader :rules, :name
46
77
  def initialize(name, &defn)
47
78
  @name = name
48
79
  @defn = defn
49
80
  @rules = []
50
81
  @loaded = false
82
+ @closed = false
51
83
  end
52
84
 
53
85
  def to_state(lexer_class)
@@ -95,10 +127,14 @@ module Rouge
95
127
  # {RegexLexer#token}, and {RegexLexer#delegate}. The first
96
128
  # argument can be used to access the match groups.
97
129
  def rule(re, tok=nil, next_state=nil, &callback)
130
+ raise ClosedState.new(self) if @closed
131
+
98
132
  if tok.nil? && callback.nil?
99
133
  raise "please pass `rule` a token to yield or a callback"
100
134
  end
101
135
 
136
+ matches_empty = re =~ ''
137
+
102
138
  callback ||= case next_state
103
139
  when :pop!
104
140
  proc do |stream|
@@ -123,6 +159,9 @@ module Rouge
123
159
  @stack.push(state)
124
160
  end
125
161
  when nil
162
+ # cannot use an empty-matching regexp with no predicate
163
+ raise InvalidRegex.new(re) if matches_empty
164
+
126
165
  proc do |stream|
127
166
  puts " yielding: #{tok.qualname}, #{stream[0].inspect}" if @debug
128
167
  @output_stream.call(tok, stream[0])
@@ -132,6 +171,22 @@ module Rouge
132
171
  end
133
172
 
134
173
  rules << Rule.new(re, callback)
174
+
175
+ close! if matches_empty && !context_sensitive?(re)
176
+ end
177
+
178
+ def context_sensitive?(re)
179
+ source = re.source
180
+ return true if source =~ /[(][?]<?[!=]/
181
+
182
+ # anchors count as lookahead/behind
183
+ return true if source =~ /[$^]/
184
+
185
+ false
186
+ end
187
+
188
+ def close!
189
+ @closed = true
135
190
  end
136
191
 
137
192
  # Mix in the rules from another state into this state. The rules
@@ -3,6 +3,6 @@
3
3
 
4
4
  module Rouge
5
5
  def self.version
6
- "3.23.0"
6
+ "3.24.0"
7
7
  end
8
8
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rouge
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.23.0
4
+ version: 3.24.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeanine Adkisson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-09-08 00:00:00.000000000 Z
11
+ date: 2020-10-13 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Rouge aims to a be a simple, easy-to-extend drop-in replacement for pygments.
14
14
  email:
@@ -73,6 +73,7 @@ files:
73
73
  - lib/rouge/demos/eiffel
74
74
  - lib/rouge/demos/elixir
75
75
  - lib/rouge/demos/elm
76
+ - lib/rouge/demos/email
76
77
  - lib/rouge/demos/epp
77
78
  - lib/rouge/demos/erb
78
79
  - lib/rouge/demos/erlang
@@ -109,6 +110,7 @@ files:
109
110
  - lib/rouge/demos/irb
110
111
  - lib/rouge/demos/irb_output
111
112
  - lib/rouge/demos/isbl
113
+ - lib/rouge/demos/j
112
114
  - lib/rouge/demos/janet
113
115
  - lib/rouge/demos/java
114
116
  - lib/rouge/demos/javascript
@@ -228,6 +230,7 @@ files:
228
230
  - lib/rouge/formatters/html.rb
229
231
  - lib/rouge/formatters/html_inline.rb
230
232
  - lib/rouge/formatters/html_legacy.rb
233
+ - lib/rouge/formatters/html_line_highlighter.rb
231
234
  - lib/rouge/formatters/html_line_table.rb
232
235
  - lib/rouge/formatters/html_linewise.rb
233
236
  - lib/rouge/formatters/html_pygments.rb
@@ -296,6 +299,7 @@ files:
296
299
  - lib/rouge/lexers/eiffel.rb
297
300
  - lib/rouge/lexers/elixir.rb
298
301
  - lib/rouge/lexers/elm.rb
302
+ - lib/rouge/lexers/email.rb
299
303
  - lib/rouge/lexers/epp.rb
300
304
  - lib/rouge/lexers/erb.rb
301
305
  - lib/rouge/lexers/erlang.rb
@@ -333,6 +337,7 @@ files:
333
337
  - lib/rouge/lexers/irb.rb
334
338
  - lib/rouge/lexers/isbl.rb
335
339
  - lib/rouge/lexers/isbl/builtins.rb
340
+ - lib/rouge/lexers/j.rb
336
341
  - lib/rouge/lexers/janet.rb
337
342
  - lib/rouge/lexers/java.rb
338
343
  - lib/rouge/lexers/javascript.rb