coderay 0.7.2.176 → 0.7.4.196

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. data/FOLDERS +1 -1
  2. data/README +16 -9
  3. data/lib/coderay.rb +2 -2
  4. data/lib/coderay/encoder.rb +1 -1
  5. data/lib/coderay/helpers/plugin.rb +4 -4
  6. data/lib/coderay/scanners/ruby.rb +22 -19
  7. data/lib/coderay/scanners/ruby/patterns.rb +15 -1
  8. metadata +64 -87
  9. data/demo/demo_cache.rb +0 -12
  10. data/demo/demo_count.rb +0 -10
  11. data/demo/demo_css.rb +0 -4
  12. data/demo/demo_div.rb +0 -19
  13. data/demo/demo_dump.rb +0 -15
  14. data/demo/demo_encoder.rb +0 -39
  15. data/demo/demo_global_vars.rb +0 -13
  16. data/demo/demo_global_vars2.rb +0 -28
  17. data/demo/demo_highlight.rb +0 -14
  18. data/demo/demo_html.rb +0 -394
  19. data/demo/demo_html2.rb +0 -11
  20. data/demo/demo_html_list.rb +0 -12
  21. data/demo/demo_load_encoder.rb +0 -25
  22. data/demo/demo_load_scanner.rb +0 -25
  23. data/demo/demo_more.rb +0 -205
  24. data/demo/demo_scanner.rb +0 -36
  25. data/demo/demo_server.rb +0 -110
  26. data/demo/demo_simple.rb +0 -10
  27. data/demo/demo_stream.rb +0 -25
  28. data/demo/demo_stream2.rb +0 -8
  29. data/demo/demo_tokens.rb +0 -3
  30. data/demo/suite.rb +0 -82
data/demo/demo_count.rb DELETED
@@ -1,10 +0,0 @@
1
- require 'coderay'
2
-
3
- stats = CodeRay.encoder(:statistic)
4
- stats.encode("puts 17 + 4\n", :ruby)
5
-
6
- puts '%d out of %d tokens have the kind :integer.' % [
7
- stats.type_stats[:integer].count,
8
- stats.real_token_count
9
- ]
10
- #-> 2 out of 4 tokens have the kind :integer.
data/demo/demo_css.rb DELETED
@@ -1,4 +0,0 @@
1
- require 'coderay'
2
-
3
- # print the default stylesheet for HTML codes
4
- puts CodeRay::Encoders[:html]::CSS.new.stylesheet
data/demo/demo_div.rb DELETED
@@ -1,19 +0,0 @@
1
- require 'coderay'
2
-
3
- puts CodeRay.scan(DATA.read, :ruby).div
4
-
5
- __END__
6
- for a in 0..255
7
- a = a.chr
8
- begin
9
- x = eval("?\\#{a}")
10
- if x == a[0]
11
- next
12
- else
13
- print "#{a}: #{x}"
14
- end
15
- rescue SyntaxError => boom
16
- print "#{a}: error"
17
- end
18
- puts
19
- end
data/demo/demo_dump.rb DELETED
@@ -1,15 +0,0 @@
1
- require 'coderay'
2
-
3
- # scan some code
4
- tokens = CodeRay.scan(File.read($0), :ruby)
5
-
6
- # dump using YAML
7
- yaml = tokens.yaml
8
- puts 'YAML: %4d bytes' % yaml.size
9
-
10
- # dump using Marshal
11
- dump = tokens.dump(0)
12
- puts 'Dump: %4d bytes' % dump.size
13
-
14
- # undump and encode
15
- puts 'undumped:', dump.undump.div(:css => :class)
data/demo/demo_encoder.rb DELETED
@@ -1,39 +0,0 @@
1
- require 'coderay'
2
-
3
- SAMPLE = "puts 17 + 4\n"
4
- puts 'Encoders Demo: ' + SAMPLE
5
- scanner = CodeRay::Scanners[:ruby].new SAMPLE
6
- encoder = CodeRay::Encoders[:statistic].new
7
-
8
- tokens = scanner.tokenize
9
- stats = encoder.encode_tokens tokens
10
-
11
- puts
12
- puts 'Statistic:'
13
- puts stats
14
-
15
- # alternative 1
16
- tokens = CodeRay.scan SAMPLE, :ruby
17
- encoder = CodeRay.encoder(:tokens)
18
- textual = encoder.encode_tokens tokens
19
- puts
20
- puts 'Original text:'
21
- puts textual
22
-
23
- # alternative 2
24
- yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby
25
- puts
26
- puts 'YAML:'
27
- puts yaml
28
-
29
- # alternative 3
30
- BIGSAMPLE = SAMPLE * 100
31
- dump = CodeRay.scan(BIGSAMPLE, :ruby).dump
32
- puts
33
- puts 'Dump:'
34
- p dump
35
- puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]
36
-
37
- puts
38
- puts 'Undump:'
39
- puts dump.undump.statistic
data/demo/demo_global_vars.rb DELETED
@@ -1,13 +0,0 @@
1
- code = <<'CODE'
2
- $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
3
- oder
4
- text = $bla.test(...)
5
- CODE
6
-
7
- require 'coderay'
8
-
9
- tokens = CodeRay.scan code, :ruby
10
- tokens.each_text_token { |text, kind| text.upcase! }
11
- tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text }
12
-
13
- print tokens
data/demo/demo_global_vars2.rb DELETED
@@ -1,28 +0,0 @@
1
- require 'coderay'
2
- require 'erb'
3
- include ERB::Util
4
-
5
- code = <<'CODE'
6
- $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
7
- oder
8
- text = $bla.test(...)
9
- CODE
10
- puts <<HTML
11
- <html>
12
- <head>
13
- <style>span.glob-var { color: green; font-weight: bold; }</style>
14
- </head>
15
- <body>
16
- HTML
17
-
18
- CodeRay.scan_stream code, :ruby do |text, kind|
19
- next if text.is_a? Symbol
20
- text = h(text)
21
- text = '<span class="glob-var">%s</span>' % text if kind == :global_variable
22
- print text
23
- end
24
-
25
- puts <<HTML
26
- </body>
27
- </html>
28
- HTML
data/demo/demo_highlight.rb DELETED
@@ -1,14 +0,0 @@
1
- require 'coderay'
2
-
3
- puts CodeRay.highlight('puts "Hello, World!"', :ruby)
4
-
5
- output = CodeRay.highlight_file($0, :line_numbers => :table)
6
- puts <<HTML
7
- <html>
8
- <head>
9
- #{output.stylesheet true}
10
- <body>
11
- #{output}
12
- </body>
13
- </html>
14
- HTML
data/demo/demo_html.rb DELETED
@@ -1,394 +0,0 @@
1
- $: << '..'
2
- require 'coderay'
3
-
4
- tokens = CodeRay.scan DATA.read, :ruby
5
- html = tokens.html(:tab_width => 2, :line_numbers => :table)
6
-
7
- puts html.page
8
-
9
- __END__
10
- require 'scanner'
11
-
12
- module CodeRay
13
-
14
- class RubyScanner < Scanner
15
-
16
- RESERVED_WORDS = [
17
- 'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
18
- 'defined?', 'ensure', 'module', 'redo', 'super', 'until',
19
- 'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
20
- 'when', 'END', 'case', 'else', 'for', 'retry',
21
- 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
22
- 'undef', 'yield',
23
- ]
24
-
25
- DEF_KEYWORDS = ['def']
26
- MODULE_KEYWORDS = ['class', 'module']
27
- DEF_NEW_STATE = WordList.new(:initial).
28
- add(DEF_KEYWORDS, :def_expected).
29
- add(MODULE_KEYWORDS, :module_expected)
30
-
31
- WORDS_ALLOWING_REGEXP = [
32
- 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
33
- ]
34
- REGEXP_ALLOWED = WordList.new(false).
35
- add(WORDS_ALLOWING_REGEXP, :set)
36
-
37
- PREDEFINED_CONSTANTS = [
38
- 'nil', 'true', 'false', 'self',
39
- 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
40
- ]
41
-
42
- IDENT_KIND = WordList.new(:ident).
43
- add(RESERVED_WORDS, :reserved).
44
- add(PREDEFINED_CONSTANTS, :pre_constant)
45
-
46
- METHOD_NAME = / #{IDENT} [?!]? /xo
47
- METHOD_NAME_EX = /
48
- #{METHOD_NAME} # common methods: split, foo=, empty?, gsub!
49
- | \*\*? # multiplication and power
50
- | [-+~]@? # plus, minus
51
- | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system`
52
- | \[\]=? # array getter and setter
53
- | <=?>? | >=? # comparison, rocket operator
54
- | << | >> # append or shift left, shift right
55
- | ===? # simple equality and case equality
56
- /ox
57
- GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
58
-
59
- DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox
60
- SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox
61
- STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox
62
- SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox
63
- REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
64
-
65
- DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error
66
- OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
67
- HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
68
- BINARY = /0b[01]+(?:_[01]+)*/
69
-
70
- EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
71
- FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
72
- INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
73
-
74
- def reset
75
- super
76
- @regexp_allowed = false
77
- end
78
-
79
- def next_token
80
- return if @scanner.eos?
81
-
82
- kind = :error
83
- if @scanner.scan(/\s+/) # in every state
84
- kind = :space
85
- @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting
86
-
87
- elsif @state == :def_expected
88
- if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
89
- kind = :method
90
- @state = :initial
91
- else
92
- @scanner.scan(/./)
93
- kind = :error
94
- end
95
- @state = :initial
96
-
97
- elsif @state == :module_expected
98
- if @scanner.scan(/<</)
99
- kind = :operator
100
- else
101
- if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
102
- kind = :method
103
- else
104
- @scanner.scan(/./)
105
- kind = :error
106
- end
107
- @state = :initial
108
- end
109
-
110
- elsif # state == :initial
111
- # IDENTIFIERS, KEYWORDS
112
- if @scanner.scan(GLOBAL_VARIABLE)
113
- kind = :global_variable
114
- elsif @scanner.scan(/ @@ #{IDENT} /ox)
115
- kind = :class_variable
116
- elsif @scanner.scan(/ @ #{IDENT} /ox)
117
- kind = :instance_variable
118
- elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
119
- kind = :comment
120
- elsif @scanner.scan(METHOD_NAME)
121
- if @last_token_dot
122
- kind = :ident
123
- else
124
- matched = @scanner.matched
125
- kind = IDENT_KIND[matched]
126
- if kind == :ident and matched =~ /^[A-Z]/
127
- kind = :constant
128
- elsif kind == :reserved
129
- @state = DEF_NEW_STATE[matched]
130
- @regexp_allowed = REGEXP_ALLOWED[matched]
131
- end
132
- end
133
-
134
- elsif @scanner.scan(STRING)
135
- kind = :string
136
- elsif @scanner.scan(SHELL)
137
- kind = :shell
138
- ## HEREDOCS
139
- elsif @scanner.scan(/\//) and @regexp_allowed
140
- @scanner.unscan
141
- @scanner.scan(REGEXP)
142
- kind = :regexp
143
- ## %strings
144
- elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
145
- kind = :global_variable
146
- elsif @scanner.scan(/
147
- \? (?:
148
- [^\s\\]
149
- |
150
- \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
151
- )
152
- /ox)
153
- kind = :integer
154
-
155
- elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
156
- kind = :operator
157
- @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
158
- elsif @scanner.scan(FLOAT)
159
- kind = :float
160
- elsif @scanner.scan(INTEGER)
161
- kind = :integer
162
- elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
163
- kind = :global_variable
164
- else
165
- @scanner.scan(/./m)
166
- end
167
- end
168
-
169
- token = Token.new @scanner.matched, kind
170
-
171
- if kind == :regexp
172
- token.text << @scanner.scan(/[eimnosux]*/)
173
- end
174
-
175
- @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting
176
-
177
- token
178
- end
179
- end
180
-
181
- ScannerList.register RubyScanner, 'ruby'
182
-
183
- end
184
-
185
- module CodeRay
186
- require 'scanner'
187
-
188
- class Highlighter
189
-
190
- def initialize lang
191
- @scanner = Scanner[lang].new
192
- end
193
-
194
- def highlight code
195
- @scanner.feed code
196
- @scanner.all_tokens.map { |t| t.inspect }.join "\n"
197
- end
198
-
199
- end
200
-
201
- class HTMLHighlighter < Highlighter
202
-
203
- ClassOfKind = {
204
- :attribute_name => 'an',
205
- :attribute_name_fat => 'af',
206
- :attribute_value => 'av',
207
- :attribute_value_fat => 'aw',
208
- :bin => 'bi',
209
- :char => 'ch',
210
- :class => 'cl',
211
- :class_variable => 'cv',
212
- :color => 'cr',
213
- :comment => 'c',
214
- :constant => 'co',
215
- :definition => 'df',
216
- :directive => 'di',
217
- :doc => 'do',
218
- :doc_string => 'ds',
219
- :exception => 'ex',
220
- :error => 'er',
221
- :float => 'fl',
222
- :function => 'fu',
223
- :global_variable => 'gv',
224
- :hex => 'hx',
225
- :include => 'ic',
226
- :instance_variable => 'iv',
227
- :integer => 'i',
228
- :interpreted => 'in',
229
- :label => 'la',
230
- :local_variable => 'lv',
231
- :oct => 'oc',
232
- :operator_name => 'on',
233
- :pre_constant => 'pc',
234
- :pre_type => 'pt',
235
- :predefined => 'pd',
236
- :preprocessor => 'pp',
237
- :regexp => 'rx',
238
- :reserved => 'r',
239
- :shell => 'sh',
240
- :string => 's',
241
- :symbol => 'sy',
242
- :tag => 'ta',
243
- :tag_fat => 'tf',
244
- :tag_special => 'ts',
245
- :type => 'ty',
246
- :variable => 'v',
247
- :xml_text => 'xt',
248
-
249
- :ident => :NO_HIGHLIGHT,
250
- :operator => :NO_HIGHLIGHT,
251
- :space => :NO_HIGHLIGHT,
252
- }
253
- ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
254
- ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
255
-
256
- def initialize lang, options = {}
257
- super lang
258
-
259
- @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
260
- case level = options.fetch(:level, 'xhtml')
261
- when 'html'
262
- @HTML_BR = "<BR>\n"
263
- when 'xhtml'
264
- @HTML_BR = "<br />\n"
265
- else
266
- raise "Unknown HTML level: #{level}"
267
- end
268
- end
269
-
270
- def highlight code
271
- @scanner.feed code
272
-
273
- out = ''
274
- while t = @scanner.next_token
275
- warn t.inspect if t.text.nil?
276
- out << to_html(t)
277
- end
278
- TEMPLATE =~ /<%CONTENT%>/
279
- $` + out + $'
280
- end
281
-
282
- private
283
- def to_html token
284
- css_class = ClassOfKind[token.kind]
285
- if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
286
- warn "no token class found for :#{token.kind}"
287
- end
288
-
289
- text = text_to_html token.text
290
- if css_class == :NO_HIGHLIGHT
291
- text
292
- else
293
- "<span class=\"#{css_class}\">#{text}</span>"
294
- end
295
- end
296
-
297
- def text_to_html text
298
- return '' if text.empty?
299
- text = text.dup # important
300
- if text.index(/["><&]/)
301
- text.gsub!('&', '&amp;')
302
- text.gsub!('"', '&quot;')
303
- text.gsub!('>', '&gt;')
304
- text.gsub!('<', '&lt;')
305
- end
306
- if text.index(/\s/)
307
- text.gsub!("\n", @HTML_BR)
308
- text.gsub!("\t", @HTML_TAB)
309
- text.gsub!(/^ /, '&nbsp;')
310
- text.gsub!(' ', ' &nbsp;')
311
- end
312
- text
313
- end
314
-
315
- TEMPLATE = <<-'TEMPLATE'
316
- <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
317
- <html dir="ltr">
318
- <head>
319
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
320
- <meta http-equiv="Content-Style-Type" content="text/css">
321
-
322
- <title>RubyBB BBCode</title>
323
- <style type="text/css">
324
- .code {
325
- width: 100%;
326
- background-color: #FAFAFA;
327
- border: 1px solid #D1D7DC;
328
- font-family: 'Courier New', 'Terminal', monospace;
329
- font-size: 10pt;
330
- color: black;
331
- vertical-align: top;
332
- text-align: left;
333
- }
334
- .code .af { color:#00C; }
335
- .code .an { color:#007; }
336
- .code .av { color:#700; }
337
- .code .aw { color:#C00; }
338
- .code .bi { color:#509; font-weight:bold; }
339
- .code .c { color:#888; }
340
- .code .ch { color:#C28; font-weight:bold; }
341
- .code .cl { color:#B06; font-weight:bold; }
342
- .code .co { color:#036; font-weight:bold; }
343
- .code .cr { color:#0A0; }
344
- .code .cv { color:#369; }
345
- .code .df { color:#099; font-weight:bold; }
346
- .code .di { color:#088; font-weight:bold; }
347
- .code .do { color:#970; }
348
- .code .ds { color:#D42; font-weight:bold; }
349
- .code .er { color:#F00; background-color:#FAA; }
350
- .code .ex { color:#F00; font-weight:bold; }
351
- .code .fl { color:#60E; font-weight:bold; }
352
- .code .fu { color:#06B; font-weight:bold; }
353
- .code .gv { color:#800; font-weight:bold; }
354
- .code .hx { color:#058; font-weight:bold; }
355
- .code .i { color:#00D; font-weight:bold; }
356
- .code .ic { color:#B44; font-weight:bold; }
357
- .code .in { color:#B2B; font-weight:bold; }
358
- .code .iv { color:#33B; }
359
- .code .la { color:#970; font-weight:bold; }
360
- .code .lv { color:#963; }
361
- .code .oc { color:#40E; font-weight:bold; }
362
- .code .on { color:#000; font-weight:bold; }
363
- .code .pc { color:#038; font-weight:bold; }
364
- .code .pd { color:#369; font-weight:bold; }
365
- .code .pp { color:#579; }
366
- .code .pt { color:#339; font-weight:bold; }
367
- .code .r { color:#080; font-weight:bold; }
368
- .code .rx { color:#927; font-weight:bold; }
369
- .code .s { color:#D42; font-weight:bold; }
370
- .code .sh { color:#B2B; font-weight:bold; }
371
- .code .sy { color:#A60; }
372
- .code .ta { color:#070; }
373
- .code .tf { color:#070; font-weight:bold; }
374
- .code .ts { color:#D70; font-weight:bold; }
375
- .code .ty { color:#339; font-weight:bold; }
376
- .code .v { color:#036; }
377
- .code .xt { color:#444; }
378
- </style>
379
- </head>
380
- <body>
381
- <div class="code">
382
- <%CONTENT%>
383
- </div>
384
- <div class="validators">
385
- <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
386
- <img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
387
- </div>
388
- </body>
389
- </html>
390
- TEMPLATE
391
-
392
- end
393
-
394
- end