coderay 0.7.2.176 → 0.7.4.196

Sign up to get free protection for your applications and to get access to all the features.
Files changed (30) hide show
  1. data/FOLDERS +1 -1
  2. data/README +16 -9
  3. data/lib/coderay.rb +2 -2
  4. data/lib/coderay/encoder.rb +1 -1
  5. data/lib/coderay/helpers/plugin.rb +4 -4
  6. data/lib/coderay/scanners/ruby.rb +22 -19
  7. data/lib/coderay/scanners/ruby/patterns.rb +15 -1
  8. metadata +64 -87
  9. data/demo/demo_cache.rb +0 -12
  10. data/demo/demo_count.rb +0 -10
  11. data/demo/demo_css.rb +0 -4
  12. data/demo/demo_div.rb +0 -19
  13. data/demo/demo_dump.rb +0 -15
  14. data/demo/demo_encoder.rb +0 -39
  15. data/demo/demo_global_vars.rb +0 -13
  16. data/demo/demo_global_vars2.rb +0 -28
  17. data/demo/demo_highlight.rb +0 -14
  18. data/demo/demo_html.rb +0 -394
  19. data/demo/demo_html2.rb +0 -11
  20. data/demo/demo_html_list.rb +0 -12
  21. data/demo/demo_load_encoder.rb +0 -25
  22. data/demo/demo_load_scanner.rb +0 -25
  23. data/demo/demo_more.rb +0 -205
  24. data/demo/demo_scanner.rb +0 -36
  25. data/demo/demo_server.rb +0 -110
  26. data/demo/demo_simple.rb +0 -10
  27. data/demo/demo_stream.rb +0 -25
  28. data/demo/demo_stream2.rb +0 -8
  29. data/demo/demo_tokens.rb +0 -3
  30. data/demo/suite.rb +0 -82
data/demo/demo_count.rb DELETED
@@ -1,10 +0,0 @@
1
- require 'coderay'
2
-
3
- stats = CodeRay.encoder(:statistic)
4
- stats.encode("puts 17 + 4\n", :ruby)
5
-
6
- puts '%d out of %d tokens have the kind :integer.' % [
7
- stats.type_stats[:integer].count,
8
- stats.real_token_count
9
- ]
10
- #-> 2 out of 4 tokens have the kind :integer.
data/demo/demo_css.rb DELETED
@@ -1,4 +0,0 @@
1
- require 'coderay'
2
-
3
- # print the default stylesheet for HTML codes
4
- puts CodeRay::Encoders[:html]::CSS.new.stylesheet
data/demo/demo_div.rb DELETED
@@ -1,19 +0,0 @@
1
- require 'coderay'
2
-
3
- puts CodeRay.scan(DATA.read, :ruby).div
4
-
5
- __END__
6
- for a in 0..255
7
- a = a.chr
8
- begin
9
- x = eval("?\\#{a}")
10
- if x == a[0]
11
- next
12
- else
13
- print "#{a}: #{x}"
14
- end
15
- rescue SyntaxError => boom
16
- print "#{a}: error"
17
- end
18
- puts
19
- end
data/demo/demo_dump.rb DELETED
@@ -1,15 +0,0 @@
1
- require 'coderay'
2
-
3
- # scan some code
4
- tokens = CodeRay.scan(File.read($0), :ruby)
5
-
6
- # dump using YAML
7
- yaml = tokens.yaml
8
- puts 'YAML: %4d bytes' % yaml.size
9
-
10
- # dump using Marshal
11
- dump = tokens.dump(0)
12
- puts 'Dump: %4d bytes' % dump.size
13
-
14
- # undump and encode
15
- puts 'undumped:', dump.undump.div(:css => :class)
data/demo/demo_encoder.rb DELETED
@@ -1,39 +0,0 @@
1
- require 'coderay'
2
-
3
- SAMPLE = "puts 17 + 4\n"
4
- puts 'Encoders Demo: ' + SAMPLE
5
- scanner = CodeRay::Scanners[:ruby].new SAMPLE
6
- encoder = CodeRay::Encoders[:statistic].new
7
-
8
- tokens = scanner.tokenize
9
- stats = encoder.encode_tokens tokens
10
-
11
- puts
12
- puts 'Statistic:'
13
- puts stats
14
-
15
- # alternative 1
16
- tokens = CodeRay.scan SAMPLE, :ruby
17
- encoder = CodeRay.encoder(:tokens)
18
- textual = encoder.encode_tokens tokens
19
- puts
20
- puts 'Original text:'
21
- puts textual
22
-
23
- # alternative 2
24
- yaml = CodeRay.encoder(:yaml).encode SAMPLE, :ruby
25
- puts
26
- puts 'YAML:'
27
- puts yaml
28
-
29
- # alternative 3
30
- BIGSAMPLE = SAMPLE * 100
31
- dump = CodeRay.scan(BIGSAMPLE, :ruby).dump
32
- puts
33
- puts 'Dump:'
34
- p dump
35
- puts 'compressed: %d byte < %d byte' % [dump.size, BIGSAMPLE.size]
36
-
37
- puts
38
- puts 'Undump:'
39
- puts dump.undump.statistic
data/demo/demo_global_vars.rb DELETED
@@ -1,13 +0,0 @@
1
- code = <<'CODE'
2
- $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
3
- oder
4
- text = $bla.test(...)
5
- CODE
6
-
7
- require 'coderay'
8
-
9
- tokens = CodeRay.scan code, :ruby
10
- tokens.each_text_token { |text, kind| text.upcase! }
11
- tokens.each(:global_variable) { |text, kind| text.replace '<--%s-->' % text }
12
-
13
- print tokens
data/demo/demo_global_vars2.rb DELETED
@@ -1,28 +0,0 @@
1
- require 'coderay'
2
- require 'erb'
3
- include ERB::Util
4
-
5
- code = <<'CODE'
6
- $ie.text_field(:name, "pAnfrage ohne $gV und mit #{$gv}").set artikel
7
- oder
8
- text = $bla.test(...)
9
- CODE
10
- puts <<HTML
11
- <html>
12
- <head>
13
- <style>span.glob-var { color: green; font-weight: bold; }</style>
14
- </head>
15
- <body>
16
- HTML
17
-
18
- CodeRay.scan_stream code, :ruby do |text, kind|
19
- next if text.is_a? Symbol
20
- text = h(text)
21
- text = '<span class="glob-var">%s</span>' % text if kind == :global_variable
22
- print text
23
- end
24
-
25
- puts <<HTML
26
- </body>
27
- </html>
28
- HTML
data/demo/demo_highlight.rb DELETED
@@ -1,14 +0,0 @@
1
- require 'coderay'
2
-
3
- puts CodeRay.highlight('puts "Hello, World!"', :ruby)
4
-
5
- output = CodeRay.highlight_file($0, :line_numbers => :table)
6
- puts <<HTML
7
- <html>
8
- <head>
9
- #{output.stylesheet true}
10
- <body>
11
- #{output}
12
- </body>
13
- </html>
14
- HTML
data/demo/demo_html.rb DELETED
@@ -1,394 +0,0 @@
1
- $: << '..'
2
- require 'coderay'
3
-
4
- tokens = CodeRay.scan DATA.read, :ruby
5
- html = tokens.html(:tab_width => 2, :line_numbers => :table)
6
-
7
- puts html.page
8
-
9
- __END__
10
- require 'scanner'
11
-
12
- module CodeRay
13
-
14
- class RubyScanner < Scanner
15
-
16
- RESERVED_WORDS = [
17
- 'and', 'def', 'end', 'in', 'or', 'unless', 'begin',
18
- 'defined?', 'ensure', 'module', 'redo', 'super', 'until',
19
- 'BEGIN', 'break', 'do', 'next', 'rescue', 'then',
20
- 'when', 'END', 'case', 'else', 'for', 'retry',
21
- 'while', 'alias', 'class', 'elsif', 'if', 'not', 'return',
22
- 'undef', 'yield',
23
- ]
24
-
25
- DEF_KEYWORDS = ['def']
26
- MODULE_KEYWORDS = ['class', 'module']
27
- DEF_NEW_STATE = WordList.new(:initial).
28
- add(DEF_KEYWORDS, :def_expected).
29
- add(MODULE_KEYWORDS, :module_expected)
30
-
31
- WORDS_ALLOWING_REGEXP = [
32
- 'and', 'or', 'not', 'while', 'until', 'unless', 'if', 'elsif', 'when'
33
- ]
34
- REGEXP_ALLOWED = WordList.new(false).
35
- add(WORDS_ALLOWING_REGEXP, :set)
36
-
37
- PREDEFINED_CONSTANTS = [
38
- 'nil', 'true', 'false', 'self',
39
- 'DATA', 'ARGV', 'ARGF', '__FILE__', '__LINE__',
40
- ]
41
-
42
- IDENT_KIND = WordList.new(:ident).
43
- add(RESERVED_WORDS, :reserved).
44
- add(PREDEFINED_CONSTANTS, :pre_constant)
45
-
46
- METHOD_NAME = / #{IDENT} [?!]? /xo
47
- METHOD_NAME_EX = /
48
- #{METHOD_NAME} # common methods: split, foo=, empty?, gsub!
49
- | \*\*? # multiplication and power
50
- | [-+~]@? # plus, minus
51
- | [\/%&|^`] # division, modulo or format strings, &and, |or, ^xor, `system`
52
- | \[\]=? # array getter and setter
53
- | <=?>? | >=? # comparison, rocket operator
54
- | << | >> # append or shift left, shift right
55
- | ===? # simple equality and case equality
56
- /ox
57
- GLOBAL_VARIABLE = / \$ (?: #{IDENT} | \d+ | [~&+`'=\/,;_.<>!@0$?*":F\\] | -[a-zA-Z_0-9] ) /ox
58
-
59
- DOUBLEQ = / " [^"\#\\]* (?: (?: \#\{.*?\} | \#(?:$")? | \\. ) [^"\#\\]* )* "? /ox
60
- SINGLEQ = / ' [^'\\]* (?: \\. [^'\\]* )* '? /ox
61
- STRING = / #{SINGLEQ} | #{DOUBLEQ} /ox
62
- SHELL = / ` [^`\#\\]* (?: (?: \#\{.*?\} | \#(?:$`)? | \\. ) [^`\#\\]* )* `? /ox
63
- REGEXP = / \/ [^\/\#\\]* (?: (?: \#\{.*?\} | \#(?:$\/)? | \\. ) [^\/\#\\]* )* \/? /ox
64
-
65
- DECIMAL = /\d+(?:_\d+)*/ # doesn't recognize 09 as octal error
66
- OCTAL = /0_?[0-7]+(?:_[0-7]+)*/
67
- HEXADECIMAL = /0x[0-9A-Fa-f]+(?:_[0-9A-Fa-f]+)*/
68
- BINARY = /0b[01]+(?:_[01]+)*/
69
-
70
- EXPONENT = / [eE] [+-]? #{DECIMAL} /ox
71
- FLOAT = / #{DECIMAL} (?: #{EXPONENT} | \. #{DECIMAL} #{EXPONENT}? ) /
72
- INTEGER = /#{OCTAL}|#{HEXADECIMAL}|#{BINARY}|#{DECIMAL}/
73
-
74
- def reset
75
- super
76
- @regexp_allowed = false
77
- end
78
-
79
- def next_token
80
- return if @scanner.eos?
81
-
82
- kind = :error
83
- if @scanner.scan(/\s+/) # in every state
84
- kind = :space
85
- @regexp_allowed = :set if @regexp_allowed or @scanner.matched.index(?\n) # delayed flag setting
86
-
87
- elsif @state == :def_expected
88
- if @scanner.scan(/ (?: (?:#{IDENT}(?:\.|::))* | (?:@@?|$)? #{IDENT}(?:\.|::) ) #{METHOD_NAME_EX} /ox)
89
- kind = :method
90
- @state = :initial
91
- else
92
- @scanner.scan(/./)
93
- kind = :error
94
- end
95
- @state = :initial
96
-
97
- elsif @state == :module_expected
98
- if @scanner.scan(/<</)
99
- kind = :operator
100
- else
101
- if @scanner.scan(/ (?: #{IDENT} (?:\.|::))* #{IDENT} /ox)
102
- kind = :method
103
- else
104
- @scanner.scan(/./)
105
- kind = :error
106
- end
107
- @state = :initial
108
- end
109
-
110
- elsif # state == :initial
111
- # IDENTIFIERS, KEYWORDS
112
- if @scanner.scan(GLOBAL_VARIABLE)
113
- kind = :global_variable
114
- elsif @scanner.scan(/ @@ #{IDENT} /ox)
115
- kind = :class_variable
116
- elsif @scanner.scan(/ @ #{IDENT} /ox)
117
- kind = :instance_variable
118
- elsif @scanner.scan(/ __END__\n ( (?!\#CODE\#) .* )? | \#[^\n]* | =begin(?=\s).*? \n=end(?=\s|\z)(?:[^\n]*)? /x)
119
- kind = :comment
120
- elsif @scanner.scan(METHOD_NAME)
121
- if @last_token_dot
122
- kind = :ident
123
- else
124
- matched = @scanner.matched
125
- kind = IDENT_KIND[matched]
126
- if kind == :ident and matched =~ /^[A-Z]/
127
- kind = :constant
128
- elsif kind == :reserved
129
- @state = DEF_NEW_STATE[matched]
130
- @regexp_allowed = REGEXP_ALLOWED[matched]
131
- end
132
- end
133
-
134
- elsif @scanner.scan(STRING)
135
- kind = :string
136
- elsif @scanner.scan(SHELL)
137
- kind = :shell
138
- ## HEREDOCS
139
- elsif @scanner.scan(/\//) and @regexp_allowed
140
- @scanner.unscan
141
- @scanner.scan(REGEXP)
142
- kind = :regexp
143
- ## %strings
144
- elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
145
- kind = :global_variable
146
- elsif @scanner.scan(/
147
- \? (?:
148
- [^\s\\]
149
- |
150
- \\ (?:M-\\C-|C-\\M-|M-\\c|c\\M-|c|C-|M-))? (?: \\ (?: . | [0-7]{3} | x[0-9A-Fa-f][0-9A-Fa-f] )
151
- )
152
- /ox)
153
- kind = :integer
154
-
155
- elsif @scanner.scan(/ [-+*\/%=<>;,|&!()\[\]{}~?] | \.\.?\.? | ::? /x)
156
- kind = :operator
157
- @regexp_allowed = :set if @scanner.matched[-1,1] =~ /[~=!<>|&^,\(\[+\-\/\*%]\z/
158
- elsif @scanner.scan(FLOAT)
159
- kind = :float
160
- elsif @scanner.scan(INTEGER)
161
- kind = :integer
162
- elsif @scanner.scan(/:(?:#{GLOBAL_VARIABLE}|#{METHOD_NAME_EX}|#{STRING})/ox)
163
- kind = :global_variable
164
- else
165
- @scanner.scan(/./m)
166
- end
167
- end
168
-
169
- token = Token.new @scanner.matched, kind
170
-
171
- if kind == :regexp
172
- token.text << @scanner.scan(/[eimnosux]*/)
173
- end
174
-
175
- @regexp_allowed = (@regexp_allowed == :set) # delayed flag setting
176
-
177
- token
178
- end
179
- end
180
-
181
- ScannerList.register RubyScanner, 'ruby'
182
-
183
- end
184
-
185
- module CodeRay
186
- require 'scanner'
187
-
188
- class Highlighter
189
-
190
- def initialize lang
191
- @scanner = Scanner[lang].new
192
- end
193
-
194
- def highlight code
195
- @scanner.feed code
196
- @scanner.all_tokens.map { |t| t.inspect }.join "\n"
197
- end
198
-
199
- end
200
-
201
- class HTMLHighlighter < Highlighter
202
-
203
- ClassOfKind = {
204
- :attribute_name => 'an',
205
- :attribute_name_fat => 'af',
206
- :attribute_value => 'av',
207
- :attribute_value_fat => 'aw',
208
- :bin => 'bi',
209
- :char => 'ch',
210
- :class => 'cl',
211
- :class_variable => 'cv',
212
- :color => 'cr',
213
- :comment => 'c',
214
- :constant => 'co',
215
- :definition => 'df',
216
- :directive => 'di',
217
- :doc => 'do',
218
- :doc_string => 'ds',
219
- :exception => 'ex',
220
- :error => 'er',
221
- :float => 'fl',
222
- :function => 'fu',
223
- :global_variable => 'gv',
224
- :hex => 'hx',
225
- :include => 'ic',
226
- :instance_variable => 'iv',
227
- :integer => 'i',
228
- :interpreted => 'in',
229
- :label => 'la',
230
- :local_variable => 'lv',
231
- :oct => 'oc',
232
- :operator_name => 'on',
233
- :pre_constant => 'pc',
234
- :pre_type => 'pt',
235
- :predefined => 'pd',
236
- :preprocessor => 'pp',
237
- :regexp => 'rx',
238
- :reserved => 'r',
239
- :shell => 'sh',
240
- :string => 's',
241
- :symbol => 'sy',
242
- :tag => 'ta',
243
- :tag_fat => 'tf',
244
- :tag_special => 'ts',
245
- :type => 'ty',
246
- :variable => 'v',
247
- :xml_text => 'xt',
248
-
249
- :ident => :NO_HIGHLIGHT,
250
- :operator => :NO_HIGHLIGHT,
251
- :space => :NO_HIGHLIGHT,
252
- }
253
- ClassOfKind[:procedure] = ClassOfKind[:method] = ClassOfKind[:function]
254
- ClassOfKind.default = ClassOfKind[:error] or raise 'no class found for :error!'
255
-
256
- def initialize lang, options = {}
257
- super lang
258
-
259
- @HTML_TAB = ' ' * options.fetch(:tabs2space, 8)
260
- case level = options.fetch(:level, 'xhtml')
261
- when 'html'
262
- @HTML_BR = "<BR>\n"
263
- when 'xhtml'
264
- @HTML_BR = "<br />\n"
265
- else
266
- raise "Unknown HTML level: #{level}"
267
- end
268
- end
269
-
270
- def highlight code
271
- @scanner.feed code
272
-
273
- out = ''
274
- while t = @scanner.next_token
275
- warn t.inspect if t.text.nil?
276
- out << to_html(t)
277
- end
278
- TEMPLATE =~ /<%CONTENT%>/
279
- $` + out + $'
280
- end
281
-
282
- private
283
- def to_html token
284
- css_class = ClassOfKind[token.kind]
285
- if defined? ::DEBUG and not ClassOfKind.has_key? token.kind
286
- warn "no token class found for :#{token.kind}"
287
- end
288
-
289
- text = text_to_html token.text
290
- if css_class == :NO_HIGHLIGHT
291
- text
292
- else
293
- "<span class=\"#{css_class}\">#{text}</span>"
294
- end
295
- end
296
-
297
- def text_to_html text
298
- return '' if text.empty?
299
- text = text.dup # important
300
- if text.index(/["><&]/)
301
- text.gsub!('&', '&amp;')
302
- text.gsub!('"', '&quot;')
303
- text.gsub!('>', '&gt;')
304
- text.gsub!('<', '&lt;')
305
- end
306
- if text.index(/\s/)
307
- text.gsub!("\n", @HTML_BR)
308
- text.gsub!("\t", @HTML_TAB)
309
- text.gsub!(/^ /, '&nbsp;')
310
- text.gsub!(' ', ' &nbsp;')
311
- end
312
- text
313
- end
314
-
315
- TEMPLATE = <<-'TEMPLATE'
316
- <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
317
- <html dir="ltr">
318
- <head>
319
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
320
- <meta http-equiv="Content-Style-Type" content="text/css">
321
-
322
- <title>RubyBB BBCode</title>
323
- <style type="text/css">
324
- .code {
325
- width: 100%;
326
- background-color: #FAFAFA;
327
- border: 1px solid #D1D7DC;
328
- font-family: 'Courier New', 'Terminal', monospace;
329
- font-size: 10pt;
330
- color: black;
331
- vertical-align: top;
332
- text-align: left;
333
- }
334
- .code .af { color:#00C; }
335
- .code .an { color:#007; }
336
- .code .av { color:#700; }
337
- .code .aw { color:#C00; }
338
- .code .bi { color:#509; font-weight:bold; }
339
- .code .c { color:#888; }
340
- .code .ch { color:#C28; font-weight:bold; }
341
- .code .cl { color:#B06; font-weight:bold; }
342
- .code .co { color:#036; font-weight:bold; }
343
- .code .cr { color:#0A0; }
344
- .code .cv { color:#369; }
345
- .code .df { color:#099; font-weight:bold; }
346
- .code .di { color:#088; font-weight:bold; }
347
- .code .do { color:#970; }
348
- .code .ds { color:#D42; font-weight:bold; }
349
- .code .er { color:#F00; background-color:#FAA; }
350
- .code .ex { color:#F00; font-weight:bold; }
351
- .code .fl { color:#60E; font-weight:bold; }
352
- .code .fu { color:#06B; font-weight:bold; }
353
- .code .gv { color:#800; font-weight:bold; }
354
- .code .hx { color:#058; font-weight:bold; }
355
- .code .i { color:#00D; font-weight:bold; }
356
- .code .ic { color:#B44; font-weight:bold; }
357
- .code .in { color:#B2B; font-weight:bold; }
358
- .code .iv { color:#33B; }
359
- .code .la { color:#970; font-weight:bold; }
360
- .code .lv { color:#963; }
361
- .code .oc { color:#40E; font-weight:bold; }
362
- .code .on { color:#000; font-weight:bold; }
363
- .code .pc { color:#038; font-weight:bold; }
364
- .code .pd { color:#369; font-weight:bold; }
365
- .code .pp { color:#579; }
366
- .code .pt { color:#339; font-weight:bold; }
367
- .code .r { color:#080; font-weight:bold; }
368
- .code .rx { color:#927; font-weight:bold; }
369
- .code .s { color:#D42; font-weight:bold; }
370
- .code .sh { color:#B2B; font-weight:bold; }
371
- .code .sy { color:#A60; }
372
- .code .ta { color:#070; }
373
- .code .tf { color:#070; font-weight:bold; }
374
- .code .ts { color:#D70; font-weight:bold; }
375
- .code .ty { color:#339; font-weight:bold; }
376
- .code .v { color:#036; }
377
- .code .xt { color:#444; }
378
- </style>
379
- </head>
380
- <body>
381
- <div class="code">
382
- <%CONTENT%>
383
- </div>
384
- <div class="validators">
385
- <a href="http://validator.w3.org/check?uri=referer"><img src="http://www.w3.org/Icons/valid-html401" alt="Valid HTML 4.01!" height="31" width="88" style="border:none;"></a>
386
- <img style="border:0" src="http://jigsaw.w3.org/css-validator/images/vcss" alt="Valid CSS!" >
387
- </div>
388
- </body>
389
- </html>
390
- TEMPLATE
391
-
392
- end
393
-
394
- end