coderay-beta 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/FOLDERS +53 -0
  2. data/LICENSE +504 -0
  3. data/bin/coderay +82 -0
  4. data/bin/coderay_stylesheet +4 -0
  5. data/lib/README +129 -0
  6. data/lib/coderay.rb +320 -0
  7. data/lib/coderay/duo.rb +85 -0
  8. data/lib/coderay/encoder.rb +213 -0
  9. data/lib/coderay/encoders/_map.rb +11 -0
  10. data/lib/coderay/encoders/comment_filter.rb +43 -0
  11. data/lib/coderay/encoders/count.rb +21 -0
  12. data/lib/coderay/encoders/debug.rb +49 -0
  13. data/lib/coderay/encoders/div.rb +19 -0
  14. data/lib/coderay/encoders/filter.rb +75 -0
  15. data/lib/coderay/encoders/html.rb +305 -0
  16. data/lib/coderay/encoders/html/css.rb +70 -0
  17. data/lib/coderay/encoders/html/numerization.rb +133 -0
  18. data/lib/coderay/encoders/html/output.rb +206 -0
  19. data/lib/coderay/encoders/json.rb +69 -0
  20. data/lib/coderay/encoders/lines_of_code.rb +90 -0
  21. data/lib/coderay/encoders/null.rb +26 -0
  22. data/lib/coderay/encoders/page.rb +20 -0
  23. data/lib/coderay/encoders/span.rb +19 -0
  24. data/lib/coderay/encoders/statistic.rb +77 -0
  25. data/lib/coderay/encoders/term.rb +137 -0
  26. data/lib/coderay/encoders/text.rb +32 -0
  27. data/lib/coderay/encoders/token_class_filter.rb +84 -0
  28. data/lib/coderay/encoders/xml.rb +71 -0
  29. data/lib/coderay/encoders/yaml.rb +22 -0
  30. data/lib/coderay/for_redcloth.rb +85 -0
  31. data/lib/coderay/helpers/file_type.rb +240 -0
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +349 -0
  34. data/lib/coderay/helpers/word_list.rb +138 -0
  35. data/lib/coderay/scanner.rb +284 -0
  36. data/lib/coderay/scanners/_map.rb +23 -0
  37. data/lib/coderay/scanners/c.rb +203 -0
  38. data/lib/coderay/scanners/cpp.rb +228 -0
  39. data/lib/coderay/scanners/css.rb +210 -0
  40. data/lib/coderay/scanners/debug.rb +62 -0
  41. data/lib/coderay/scanners/delphi.rb +150 -0
  42. data/lib/coderay/scanners/diff.rb +105 -0
  43. data/lib/coderay/scanners/groovy.rb +263 -0
  44. data/lib/coderay/scanners/html.rb +182 -0
  45. data/lib/coderay/scanners/java.rb +176 -0
  46. data/lib/coderay/scanners/java/builtin_types.rb +419 -0
  47. data/lib/coderay/scanners/java_script.rb +224 -0
  48. data/lib/coderay/scanners/json.rb +112 -0
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +526 -0
  51. data/lib/coderay/scanners/plaintext.rb +21 -0
  52. data/lib/coderay/scanners/python.rb +285 -0
  53. data/lib/coderay/scanners/rhtml.rb +74 -0
  54. data/lib/coderay/scanners/ruby.rb +404 -0
  55. data/lib/coderay/scanners/ruby/patterns.rb +238 -0
  56. data/lib/coderay/scanners/scheme.rb +145 -0
  57. data/lib/coderay/scanners/sql.rb +162 -0
  58. data/lib/coderay/scanners/xml.rb +17 -0
  59. data/lib/coderay/scanners/yaml.rb +144 -0
  60. data/lib/coderay/style.rb +20 -0
  61. data/lib/coderay/styles/_map.rb +7 -0
  62. data/lib/coderay/styles/cycnus.rb +151 -0
  63. data/lib/coderay/styles/murphy.rb +132 -0
  64. data/lib/coderay/token_classes.rb +86 -0
  65. data/lib/coderay/tokens.rb +391 -0
  66. data/lib/term/ansicolor.rb +220 -0
  67. metadata +123 -0
@@ -0,0 +1,228 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class CPlusPlus < Scanner
5
+
6
+ include Streamable
7
+
8
+ register_for :cpp
9
+ file_extension 'cpp'
10
+ title 'C++'
11
+
12
+ # http://www.cppreference.com/wiki/keywords/start
13
+ RESERVED_WORDS = [
14
+ 'and', 'and_eq', 'asm', 'bitand', 'bitor', 'break',
15
+ 'case', 'catch', 'class', 'compl', 'const_cast',
16
+ 'continue', 'default', 'delete', 'do', 'dynamic_cast', 'else',
17
+ 'enum', 'export', 'for', 'goto', 'if', 'namespace', 'new',
18
+ 'not', 'not_eq', 'or', 'or_eq', 'reinterpret_cast', 'return',
19
+ 'sizeof', 'static_cast', 'struct', 'switch', 'template',
20
+ 'throw', 'try', 'typedef', 'typeid', 'typename', 'union',
21
+ 'while', 'xor', 'xor_eq'
22
+ ]
23
+
24
+ PREDEFINED_TYPES = [
25
+ 'bool', 'char', 'double', 'float', 'int', 'long',
26
+ 'short', 'signed', 'unsigned', 'wchar_t', 'string'
27
+ ]
28
+ PREDEFINED_CONSTANTS = [
29
+ 'false', 'true',
30
+ 'EOF', 'NULL',
31
+ ]
32
+ PREDEFINED_VARIABLES = [
33
+ 'this'
34
+ ]
35
+ DIRECTIVES = [
36
+ 'auto', 'const', 'explicit', 'extern', 'friend', 'inline', 'mutable', 'operator',
37
+ 'private', 'protected', 'public', 'register', 'static', 'using', 'virtual', 'void',
38
+ 'volatile'
39
+ ]
40
+
41
+ IDENT_KIND = WordList.new(:ident).
42
+ add(RESERVED_WORDS, :reserved).
43
+ add(PREDEFINED_TYPES, :pre_type).
44
+ add(PREDEFINED_VARIABLES, :local_variable).
45
+ add(DIRECTIVES, :directive).
46
+ add(PREDEFINED_CONSTANTS, :pre_constant)
47
+
48
+ ESCAPE = / [rbfntv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
49
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
50
+
51
+ def scan_tokens tokens, options
52
+
53
+ state = :initial
54
+ label_expected = true
55
+ case_expected = false
56
+ label_expected_before_preproc_line = nil
57
+ in_preproc_line = false
58
+
59
+ until eos?
60
+
61
+ kind = nil
62
+ match = nil
63
+
64
+ case state
65
+
66
+ when :initial
67
+
68
+ if match = scan(/ \s+ | \\\n /x)
69
+ if in_preproc_line && match != "\\\n" && match.index(?\n)
70
+ in_preproc_line = false
71
+ label_expected = label_expected_before_preproc_line
72
+ end
73
+ tokens << [match, :space]
74
+ next
75
+
76
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
77
+ kind = :comment
78
+
79
+ elsif match = scan(/ \# \s* if \s* 0 /x)
80
+ match << scan_until(/ ^\# (?:elif|else|endif) .*? $ | \z /xm) unless eos?
81
+ kind = :comment
82
+
83
+ elsif match = scan(/ [-+*=<>?:;,!&^|()\[\]{}~%]+ | \/=? | \.(?!\d) /x)
84
+ label_expected = match =~ /[;\{\}]/
85
+ if case_expected
86
+ label_expected = true if match == ':'
87
+ case_expected = false
88
+ end
89
+ kind = :operator
90
+
91
+ elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
92
+ kind = IDENT_KIND[match]
93
+ if kind == :ident && label_expected && !in_preproc_line && scan(/:(?!:)/)
94
+ kind = :label
95
+ match << matched
96
+ else
97
+ label_expected = false
98
+ if kind == :reserved
99
+ case match
100
+ when 'class'
101
+ state = :class_name_expected
102
+ when 'case', 'default'
103
+ case_expected = true
104
+ end
105
+ end
106
+ end
107
+
108
+ elsif scan(/\$/)
109
+ kind = :ident
110
+
111
+ elsif match = scan(/L?"/)
112
+ tokens << [:open, :string]
113
+ if match[0] == ?L
114
+ tokens << ['L', :modifier]
115
+ match = '"'
116
+ end
117
+ state = :string
118
+ kind = :delimiter
119
+
120
+ elsif scan(/#[ \t]*(\w*)/)
121
+ kind = :preprocessor
122
+ in_preproc_line = true
123
+ label_expected_before_preproc_line = label_expected
124
+ state = :include_expected if self[1] == 'include'
125
+
126
+ elsif scan(/ L?' (?: [^\'\n\\] | \\ #{ESCAPE} )? '? /ox)
127
+ label_expected = false
128
+ kind = :char
129
+
130
+ elsif scan(/0[xX][0-9A-Fa-f]+/)
131
+ label_expected = false
132
+ kind = :hex
133
+
134
+ elsif scan(/(?:0[0-7]+)(?![89.eEfF])/)
135
+ label_expected = false
136
+ kind = :oct
137
+
138
+ elsif scan(/(?:\d+)(?![.eEfF])L?L?/)
139
+ label_expected = false
140
+ kind = :integer
141
+
142
+ elsif scan(/\d[fF]?|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
143
+ label_expected = false
144
+ kind = :float
145
+
146
+ else
147
+ getch
148
+ kind = :error
149
+
150
+ end
151
+
152
+ when :string
153
+ if scan(/[^\\"]+/)
154
+ kind = :content
155
+ elsif scan(/"/)
156
+ tokens << ['"', :delimiter]
157
+ tokens << [:close, :string]
158
+ state = :initial
159
+ label_expected = false
160
+ next
161
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
162
+ kind = :char
163
+ elsif scan(/ \\ | $ /x)
164
+ tokens << [:close, :string]
165
+ kind = :error
166
+ state = :initial
167
+ label_expected = false
168
+ else
169
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
170
+ end
171
+
172
+ when :include_expected
173
+ if scan(/<[^>\n]+>?|"[^"\n\\]*(?:\\.[^"\n\\]*)*"?/)
174
+ kind = :include
175
+ state = :initial
176
+
177
+ elsif match = scan(/\s+/)
178
+ kind = :space
179
+ state = :initial if match.index ?\n
180
+
181
+ else
182
+ state = :initial
183
+ next
184
+
185
+ end
186
+
187
+ when :class_name_expected
188
+ if scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
189
+ kind = :class
190
+ state = :initial
191
+
192
+ elsif match = scan(/\s+/)
193
+ kind = :space
194
+
195
+ else
196
+ getch
197
+ kind = :error
198
+ state = :initial
199
+
200
+ end
201
+
202
+ else
203
+ raise_inspect 'Unknown state', tokens
204
+
205
+ end
206
+
207
+ match ||= matched
208
+ if $DEBUG and not kind
209
+ raise_inspect 'Error token %p in line %d' %
210
+ [[match, kind], line], tokens
211
+ end
212
+ raise_inspect 'Empty token', tokens unless match
213
+
214
+ tokens << [match, kind]
215
+
216
+ end
217
+
218
+ if state == :string
219
+ tokens << [:close, :string]
220
+ end
221
+
222
+ tokens
223
+ end
224
+
225
+ end
226
+
227
+ end
228
+ end
@@ -0,0 +1,210 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class CSS < Scanner
5
+
6
+ register_for :css
7
+
8
+ KINDS_NOT_LOC = [
9
+ :comment,
10
+ :class, :pseudo_class, :type,
11
+ :constant, :directive,
12
+ :key, :value, :operator, :color, :float,
13
+ :error, :important,
14
+ ]
15
+
16
+ module RE
17
+ NonASCII = /[\x80-\xFF]/
18
+ Hex = /[0-9a-fA-F]/
19
+ Unicode = /\\#{Hex}{1,6}(?:\r\n|\s)?/ # differs from standard because it allows uppercase hex too
20
+ Escape = /#{Unicode}|\\[^\r\n\f0-9a-fA-F]/
21
+ NMChar = /[-_a-zA-Z0-9]|#{NonASCII}|#{Escape}/
22
+ NMStart = /[_a-zA-Z]|#{NonASCII}|#{Escape}/
23
+ NL = /\r\n|\r|\n|\f/
24
+ String1 = /"(?:[^\n\r\f\\"]|\\#{NL}|#{Escape})*"?/ # FIXME: buggy regexp
25
+ String2 = /'(?:[^\n\r\f\\']|\\#{NL}|#{Escape})*'?/ # FIXME: buggy regexp
26
+ String = /#{String1}|#{String2}/
27
+
28
+ HexColor = /#(?:#{Hex}{6}|#{Hex}{3})/
29
+ Color = /#{HexColor}/
30
+
31
+ Num = /-?(?:[0-9]+|[0-9]*\.[0-9]+)/
32
+ Name = /#{NMChar}+/
33
+ Ident = /-?#{NMStart}#{NMChar}*/
34
+ AtKeyword = /@#{Ident}/
35
+ Percentage = /#{Num}%/
36
+
37
+ reldimensions = %w[em ex px]
38
+ absdimensions = %w[in cm mm pt pc]
39
+ Unit = Regexp.union(*(reldimensions + absdimensions))
40
+
41
+ Dimension = /#{Num}#{Unit}/
42
+
43
+ Comment = %r! /\* (?: .*? \*/ | .* ) !mx
44
+ Function = /(?:url|alpha)\((?:[^)\n\r\f]|\\\))*\)?/
45
+
46
+ Id = /##{Name}/
47
+ Class = /\.#{Name}/
48
+ PseudoClass = /:#{Name}/
49
+ AttributeSelector = /\[[^\]]*\]?/
50
+
51
+ end
52
+
53
+ def scan_tokens tokens, options
54
+
55
+ value_expected = nil
56
+ states = [:initial]
57
+
58
+ until eos?
59
+
60
+ kind = nil
61
+ match = nil
62
+
63
+ if scan(/\s+/)
64
+ kind = :space
65
+
66
+ elsif case states.last
67
+ when :initial, :media
68
+ if scan(/(?>#{RE::Ident})(?!\()|\*/ox)
69
+ kind = :type
70
+ elsif scan RE::Class
71
+ kind = :class
72
+ elsif scan RE::Id
73
+ kind = :constant
74
+ elsif scan RE::PseudoClass
75
+ kind = :pseudo_class
76
+ elsif match = scan(RE::AttributeSelector)
77
+ # TODO: Improve highlighting inside of attribute selectors.
78
+ tokens << [:open, :string]
79
+ tokens << [match[0,1], :delimiter]
80
+ tokens << [match[1..-2], :content] if match.size > 2
81
+ tokens << [match[-1,1], :delimiter] if match[-1] == ?]
82
+ tokens << [:close, :string]
83
+ next
84
+ elsif match = scan(/@media/)
85
+ kind = :directive
86
+ states.push :media_before_name
87
+ end
88
+
89
+ when :block
90
+ if scan(/(?>#{RE::Ident})(?!\()/ox)
91
+ if value_expected
92
+ kind = :value
93
+ else
94
+ kind = :key
95
+ end
96
+ end
97
+
98
+ when :media_before_name
99
+ if scan RE::Ident
100
+ kind = :type
101
+ states[-1] = :media_after_name
102
+ end
103
+
104
+ when :media_after_name
105
+ if scan(/\{/)
106
+ kind = :operator
107
+ states[-1] = :media
108
+ end
109
+
110
+ when :comment
111
+ if scan(/(?:[^*\s]|\*(?!\/))+/)
112
+ kind = :comment
113
+ elsif scan(/\*\//)
114
+ kind = :comment
115
+ states.pop
116
+ elsif scan(/\s+/)
117
+ kind = :space
118
+ end
119
+
120
+ else
121
+ raise_inspect 'Unknown state', tokens
122
+
123
+ end
124
+
125
+ elsif scan(/\/\*/)
126
+ kind = :comment
127
+ states.push :comment
128
+
129
+ elsif scan(/\{/)
130
+ value_expected = false
131
+ kind = :operator
132
+ states.push :block
133
+
134
+ elsif scan(/\}/)
135
+ value_expected = false
136
+ if states.last == :block || states.last == :media
137
+ kind = :operator
138
+ states.pop
139
+ else
140
+ kind = :error
141
+ end
142
+
143
+ elsif match = scan(/#{RE::String}/o)
144
+ tokens << [:open, :string]
145
+ tokens << [match[0, 1], :delimiter]
146
+ tokens << [match[1..-2], :content] if match.size > 2
147
+ tokens << [match[-1, 1], :delimiter] if match.size >= 2
148
+ tokens << [:close, :string]
149
+ next
150
+
151
+ elsif match = scan(/#{RE::Function}/o)
152
+ tokens << [:open, :string]
153
+ start = match[/^\w+\(/]
154
+ tokens << [start, :delimiter]
155
+ if match[-1] == ?)
156
+ tokens << [match[start.size..-2], :content]
157
+ tokens << [')', :delimiter]
158
+ else
159
+ tokens << [match[start.size..-1], :content]
160
+ end
161
+ tokens << [:close, :string]
162
+ next
163
+
164
+ elsif scan(/(?: #{RE::Dimension} | #{RE::Percentage} | #{RE::Num} )/ox)
165
+ kind = :float
166
+
167
+ elsif scan(/#{RE::Color}/o)
168
+ kind = :color
169
+
170
+ elsif scan(/! *important/)
171
+ kind = :important
172
+
173
+ elsif scan(/rgb\([^()\n]*\)?/)
174
+ kind = :color
175
+
176
+ elsif scan(/#{RE::AtKeyword}/o)
177
+ kind = :directive
178
+
179
+ elsif match = scan(/ [+>:;,.=()\/] /x)
180
+ if match == ':'
181
+ value_expected = true
182
+ elsif match == ';'
183
+ value_expected = false
184
+ end
185
+ kind = :operator
186
+
187
+ else
188
+ getch
189
+ kind = :error
190
+
191
+ end
192
+
193
+ match ||= matched
194
+ if $DEBUG and not kind
195
+ raise_inspect 'Error token %p in line %d' %
196
+ [[match, kind], line], tokens
197
+ end
198
+ raise_inspect 'Empty token', tokens unless match
199
+
200
+ tokens << [match, kind]
201
+
202
+ end
203
+
204
+ tokens
205
+ end
206
+
207
+ end
208
+
209
+ end
210
+ end
@@ -0,0 +1,62 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ # = Debug Scanner
5
+ class Debug < Scanner
6
+
7
+ include Streamable
8
+ register_for :debug
9
+ file_extension 'raydebug'
10
+ title 'CodeRay Token Dump'
11
+
12
+ protected
13
+ def scan_tokens tokens, options
14
+
15
+ opened_tokens = []
16
+
17
+ until eos?
18
+
19
+ kind = nil
20
+ match = nil
21
+
22
+ if scan(/\s+/)
23
+ tokens << [matched, :space]
24
+ next
25
+
26
+ elsif scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \) /x)
27
+ kind = self[1].to_sym
28
+ match = self[2].gsub(/\\(.)/, '\1')
29
+
30
+ elsif scan(/ (\w+) < /x)
31
+ kind = self[1].to_sym
32
+ opened_tokens << kind
33
+ match = :open
34
+
35
+ elsif scan(/ > /x)
36
+ kind = opened_tokens.pop
37
+ match = :close
38
+
39
+ else
40
+ kind = :error
41
+ getch
42
+
43
+ end
44
+
45
+ match ||= matched
46
+ if $DEBUG and not kind
47
+ raise_inspect 'Error token %p in line %d' %
48
+ [[match, kind], line], tokens
49
+ end
50
+ raise_inspect 'Empty token', tokens unless match
51
+
52
+ tokens << [match, kind]
53
+
54
+ end
55
+
56
+ tokens
57
+ end
58
+
59
+ end
60
+
61
+ end
62
+ end