rouge 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,300 @@
1
+ module Rouge
2
+ class RegexLexer < Lexer
3
+ class Rule
4
+ attr_reader :callback
5
+ attr_reader :next_state
6
+ attr_reader :re
7
+ def initialize(re, callback, next_state)
8
+ @re = re
9
+ @callback = callback
10
+ @next_state = next_state
11
+ end
12
+
13
+ def inspect
14
+ "#<Rule #{@re.inspect}>"
15
+ end
16
+ end
17
+
18
+ class State
19
+ attr_reader :name
20
+ def initialize(lexer_class, name, &defn)
21
+ @lexer_class = lexer_class
22
+ @name = name
23
+ @defn = defn
24
+ end
25
+
26
+ def relative_state(state_name=nil, &b)
27
+ if state_name
28
+ @lexer_class.get_state(state_name)
29
+ else
30
+ State.new(@lexer_class, b.inspect, &b).load!
31
+ end
32
+ end
33
+
34
+ def rules
35
+ @rules ||= []
36
+ end
37
+
38
+ def load!
39
+ return self if @loaded
40
+ @loaded = true
41
+ StateDSL.new(rules).instance_eval(&@defn)
42
+ self
43
+ end
44
+ end
45
+
46
+ class StateDSL
47
+ attr_reader :rules
48
+ def initialize(rules)
49
+ @rules = rules
50
+ end
51
+
52
+ def rule(re, tok=nil, next_state=nil, &callback)
53
+ if block_given?
54
+ next_state = tok
55
+ else
56
+ tok = Token[tok]
57
+
58
+ callback = proc do
59
+ token tok
60
+ case next_state
61
+ when :pop!
62
+ pop!
63
+ when Symbol
64
+ push next_state
65
+ end # else pass
66
+ end
67
+ end
68
+
69
+ rules << Rule.new(re, callback, next_state)
70
+ end
71
+
72
+ def mixin(lexer_name)
73
+ rules << lexer_name.to_s
74
+ end
75
+ end
76
+
77
+ def self.states
78
+ @states ||= {}
79
+ end
80
+
81
+ @start_procs = []
82
+ def self.start_procs
83
+ @start_procs ||= InheritableList.new(superclass.start_procs)
84
+ end
85
+
86
+ # Specify an action to be run every fresh lex.
87
+ #
88
+ # @example
89
+ # start { puts "I'm lexing a new string!" }
90
+ def self.start(&b)
91
+ start_procs << b
92
+ end
93
+
94
+ def self.postprocess(toktype, &b)
95
+ postprocesses << [Token[toktype], b]
96
+ end
97
+
98
+ @postprocesses = []
99
+ def self.postprocesses
100
+ @postprocesses ||= InheritableList.new(superclass.postprocesses)
101
+ end
102
+
103
+ def self.state(name, &b)
104
+ name = name.to_s
105
+ states[name] = State.new(self, name, &b)
106
+ end
107
+
108
+ def self.get_state(name)
109
+ return name if name.is_a? State
110
+
111
+ state = states[name.to_s]
112
+ raise "unknown state: #{name}" unless state
113
+ state.load!
114
+ end
115
+
116
+ def self.[](name)
117
+ get_state(name)
118
+ end
119
+
120
+ def get_state(name)
121
+ self.class.get_state(name)
122
+ end
123
+
124
+ def stack
125
+ @stack ||= [get_state(:root)]
126
+ end
127
+
128
+ def state
129
+ stack.last or raise 'empty stack!'
130
+ end
131
+
132
+ def reset!
133
+ @scan_state = nil
134
+
135
+ self.class.start_procs.each do |pr|
136
+ instance_eval(&pr)
137
+ end
138
+ end
139
+
140
+ def stream_tokens(stream, &b)
141
+ stream_without_postprocessing(stream) do |tok, val|
142
+ _, processor = self.class.postprocesses.find { |t, _| t == tok }
143
+
144
+ if processor
145
+ # TODO: DRY this up with run_callback
146
+ Enumerator.new do |y|
147
+ @output_stream = y
148
+ instance_exec(tok, val, &processor)
149
+ end.each do |newtok, newval|
150
+ yield Token[newtok], newval
151
+ end
152
+ else
153
+ yield tok, val
154
+ end
155
+ end
156
+ end
157
+
158
+ # @private
159
+ def stream_without_postprocessing(stream, &b)
160
+ until stream.eos?
161
+ debug { "lexer: #{self.class.tag}" }
162
+ debug { "stack: #{stack.map(&:name).inspect}" }
163
+ debug { "stream: #{stream.peek(20).inspect}" }
164
+ success = step(get_state(state), stream, &b)
165
+
166
+ if !success
167
+ debug { " no match, yielding Error" }
168
+ b.call(Token['Error'], stream.getch)
169
+ end
170
+ end
171
+ end
172
+
173
+ def step(state, stream, &b)
174
+ state.rules.each do |rule|
175
+ return true if run_rule(rule, stream, &b)
176
+ end
177
+
178
+ false
179
+ end
180
+
181
+ def run_rule(rule, stream, &b)
182
+ case rule
183
+ when String
184
+ debug { " entering mixin #{rule}" }
185
+ res = step(get_state(rule), stream, &b)
186
+ debug { " exiting mixin #{rule}" }
187
+ res
188
+ when Rule
189
+ debug { " trying #{rule.inspect}" }
190
+ scan(stream, rule.re) do
191
+ debug { " got #{stream[0].inspect}" }
192
+
193
+ run_callback(stream, &rule.callback).each do |tok, res|
194
+ debug { " yielding #{tok.to_s.inspect}, #{res.inspect}" }
195
+ b.call(Token[tok], res)
196
+ end
197
+ end
198
+ end
199
+ end
200
+
201
+ def run_callback(stream, &callback)
202
+ Enumerator.new do |y|
203
+ @output_stream = y
204
+ @group_count = 0
205
+ @last_match = stream
206
+ instance_exec(stream, &callback)
207
+ @last_match = nil
208
+ @output_stream = nil
209
+ end
210
+ end
211
+
212
+ MAX_NULL_STEPS = 5
213
+ def scan(scanner, re, &b)
214
+ # XXX HACK XXX
215
+ # StringScanner's implementation of ^ is b0rken.
216
+ # TODO: this doesn't cover cases like /(a|^b)/, but it's
217
+ # the most common, for now...
218
+ return false if re.source[0] == ?^ && !scanner.beginning_of_line?
219
+
220
+ @null_steps ||= 0
221
+
222
+ if @null_steps >= MAX_NULL_STEPS
223
+ debug { " too many scans without consuming the string!" }
224
+ return false
225
+ end
226
+
227
+ scanner.scan(re)
228
+
229
+ if scanner.matched?
230
+ if scanner.matched_size == 0
231
+ @null_steps += 1
232
+ else
233
+ @null_steps = 0
234
+ end
235
+
236
+ yield self
237
+ return true
238
+ end
239
+
240
+ return false
241
+ end
242
+
243
+ def token(tok, val=:__absent__)
244
+ val = @last_match[0] if val == :__absent__
245
+ val ||= ''
246
+
247
+ raise 'no output stream' unless @output_stream
248
+
249
+ @output_stream << [Token[tok], val] unless val.empty?
250
+ end
251
+
252
+ def group(tok)
253
+ token(tok, @last_match[@group_count += 1])
254
+ end
255
+
256
+ def delegate(lexer, text=nil)
257
+ debug { " delegating to #{lexer.inspect}" }
258
+ text ||= @last_match[0]
259
+
260
+ lexer.lex(text, :continue => true) do |tok, val|
261
+ debug { " delegated token: #{tok.inspect}, #{val.inspect}" }
262
+ token(tok, val)
263
+ end
264
+ end
265
+
266
+ def push(state_name=nil, &b)
267
+ # use the top of the stack by default
268
+ if state_name || b
269
+ push_state = state.relative_state(state_name, &b)
270
+ else
271
+ push_state = self.state
272
+ end
273
+
274
+ debug { " pushing #{push_state.name}" }
275
+ stack.push(push_state)
276
+ end
277
+
278
+ def pop!(times=1)
279
+ raise 'empty stack!' if stack.empty?
280
+
281
+ debug { " popping stack: #{times}" }
282
+ times.times { stack.pop }
283
+ end
284
+
285
+ def reset_stack
286
+ debug { ' resetting stack' }
287
+ stack.clear
288
+ stack.push get_state(:root)
289
+ end
290
+
291
+ def in_state?(state_name)
292
+ stack.map(&:name).include? state_name.to_s
293
+ end
294
+
295
+ def state?(state_name)
296
+ state_name.to_s == state.name
297
+ end
298
+
299
+ end
300
+ end
@@ -0,0 +1,14 @@
1
+ module Rouge
2
+ class TemplateLexer < RegexLexer
3
+ def parent
4
+ return @parent if instance_variable_defined? :@parent
5
+ @parent = option(:parent) || 'html'
6
+ if @parent.is_a? String
7
+ lexer_class = Lexer.find(@parent)
8
+ @parent = lexer_class.new(self.options)
9
+ end
10
+ end
11
+
12
+ start { parent.reset! }
13
+ end
14
+ end
@@ -1,39 +1,5 @@
1
1
  module Rouge
2
2
  class Theme
3
- class InheritableHash < Hash
4
- def initialize(parent=nil)
5
- @parent = parent
6
- end
7
-
8
- def [](k)
9
- _sup = super
10
- return _sup if own_keys.include?(k)
11
-
12
- _sup || parent[k]
13
- end
14
-
15
- def parent
16
- @parent ||= {}
17
- end
18
-
19
- def include?(k)
20
- super or parent.include?(k)
21
- end
22
-
23
- def each(&b)
24
- keys.each do |k|
25
- b.call(k, self[k])
26
- end
27
- end
28
-
29
- alias own_keys keys
30
- def keys
31
- keys = own_keys.concat(parent.keys)
32
- keys.uniq!
33
- keys
34
- end
35
- end
36
-
37
3
  class Style < Hash
38
4
  def initialize(theme, hsh={})
39
5
  super()
@@ -32,13 +32,14 @@ module Rouge
32
32
  style 'Generic.Deleted', :fg => :scarletred2
33
33
  style 'Generic.Inserted', :fg => :go_get_it
34
34
  style 'Generic.Emph', :italic => true
35
- style 'Generic.Subheading', :fg => '#800080', :bold => true
36
- style 'Generic.Traceback', :fg => '#0040D0'
35
+ style 'Generic.Strong', :bold => true
36
+ style 'Generic.Traceback', :fg => :eggshell_cloud, :bg => :slate_blue
37
37
  style 'Keyword.Constant', :fg => :pink_merengue, :bold => true
38
38
  style 'Keyword.Namespace',
39
39
  'Keyword.Pseudo',
40
40
  'Keyword.Reserved',
41
- 'Generic.Heading', :fg => :schrill, :bold => true
41
+ 'Generic.Heading',
42
+ 'Generic.Subheading', :fg => :schrill, :bold => true
42
43
  style 'Keyword.Type',
43
44
  'Name.Constant',
44
45
  'Name.Class',
@@ -0,0 +1,63 @@
1
+ module Rouge
2
+ class InheritableHash < Hash
3
+ def initialize(parent=nil)
4
+ @parent = parent
5
+ end
6
+
7
+ def [](k)
8
+ _sup = super
9
+ return _sup if own_keys.include?(k)
10
+
11
+ _sup || parent[k]
12
+ end
13
+
14
+ def parent
15
+ @parent ||= {}
16
+ end
17
+
18
+ def include?(k)
19
+ super or parent.include?(k)
20
+ end
21
+
22
+ def each(&b)
23
+ keys.each do |k|
24
+ b.call(k, self[k])
25
+ end
26
+ end
27
+
28
+ alias own_keys keys
29
+ def keys
30
+ keys = own_keys.concat(parent.keys)
31
+ keys.uniq!
32
+ keys
33
+ end
34
+ end
35
+
36
+ class InheritableList
37
+ include Enumerable
38
+
39
+ def initialize(parent=nil)
40
+ @parent = parent
41
+ end
42
+
43
+ def parent
44
+ @parent ||= []
45
+ end
46
+
47
+ def each(&b)
48
+ return enum_for(:each) unless block_given?
49
+
50
+ parent.each(&b)
51
+ own_entries.each(&b)
52
+ end
53
+
54
+ def own_entries
55
+ @own_entries ||= []
56
+ end
57
+
58
+ def push(o)
59
+ own_entries << o
60
+ end
61
+ alias << push
62
+ end
63
+ end
@@ -1,5 +1,5 @@
1
1
  module Rouge
2
2
  def self.version
3
- "0.1.2"
3
+ "0.2.0"
4
4
  end
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rouge
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-09-15 00:00:00.000000000 Z
12
+ date: 2012-10-10 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: thor
@@ -36,12 +36,14 @@ extensions: []
36
36
  extra_rdoc_files: []
37
37
  files:
38
38
  - Gemfile
39
+ - lib/rouge/util.rb
39
40
  - lib/rouge/lexers/python.rb
40
41
  - lib/rouge/lexers/common_lisp.rb
41
42
  - lib/rouge/lexers/erb.rb
42
43
  - lib/rouge/lexers/scheme.rb
43
44
  - lib/rouge/lexers/xml.rb
44
45
  - lib/rouge/lexers/shell.rb
46
+ - lib/rouge/lexers/haml.rb
45
47
  - lib/rouge/lexers/ruby.rb
46
48
  - lib/rouge/lexers/make.rb
47
49
  - lib/rouge/lexers/perl.rb
@@ -53,11 +55,14 @@ files:
53
55
  - lib/rouge/lexers/tex.rb
54
56
  - lib/rouge/lexers/text.rb
55
57
  - lib/rouge/lexers/html.rb
58
+ - lib/rouge/lexers/markdown.rb
56
59
  - lib/rouge/lexers/java.rb
57
60
  - lib/rouge/lexers/php.rb
61
+ - lib/rouge/lexers/yaml.rb
58
62
  - lib/rouge/lexers/tcl.rb
59
63
  - lib/rouge/lexers/css.rb
60
64
  - lib/rouge/lexers/cpp.rb
65
+ - lib/rouge/lexers/factor.rb
61
66
  - lib/rouge/plugins/redcarpet.rb
62
67
  - lib/rouge/themes/thankful_eyes.rb
63
68
  - lib/rouge/themes/colorful.rb
@@ -68,8 +73,10 @@ files:
68
73
  - lib/rouge/text_analyzer.rb
69
74
  - lib/rouge/version.rb
70
75
  - lib/rouge/formatter.rb
76
+ - lib/rouge/regex_lexer.rb
71
77
  - lib/rouge/cli.rb
72
78
  - lib/rouge/lexer.rb
79
+ - lib/rouge/template_lexer.rb
73
80
  - lib/rouge/theme.rb
74
81
  - lib/rouge.rb
75
82
  - bin/rougify
@@ -98,3 +105,4 @@ signing_key:
98
105
  specification_version: 3
99
106
  summary: A pure-ruby colorizer based on pygments
100
107
  test_files: []
108
+ has_rdoc: