coderay-beta 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/FOLDERS +53 -0
  2. data/LICENSE +504 -0
  3. data/bin/coderay +82 -0
  4. data/bin/coderay_stylesheet +4 -0
  5. data/lib/README +129 -0
  6. data/lib/coderay.rb +320 -0
  7. data/lib/coderay/duo.rb +85 -0
  8. data/lib/coderay/encoder.rb +213 -0
  9. data/lib/coderay/encoders/_map.rb +11 -0
  10. data/lib/coderay/encoders/comment_filter.rb +43 -0
  11. data/lib/coderay/encoders/count.rb +21 -0
  12. data/lib/coderay/encoders/debug.rb +49 -0
  13. data/lib/coderay/encoders/div.rb +19 -0
  14. data/lib/coderay/encoders/filter.rb +75 -0
  15. data/lib/coderay/encoders/html.rb +305 -0
  16. data/lib/coderay/encoders/html/css.rb +70 -0
  17. data/lib/coderay/encoders/html/numerization.rb +133 -0
  18. data/lib/coderay/encoders/html/output.rb +206 -0
  19. data/lib/coderay/encoders/json.rb +69 -0
  20. data/lib/coderay/encoders/lines_of_code.rb +90 -0
  21. data/lib/coderay/encoders/null.rb +26 -0
  22. data/lib/coderay/encoders/page.rb +20 -0
  23. data/lib/coderay/encoders/span.rb +19 -0
  24. data/lib/coderay/encoders/statistic.rb +77 -0
  25. data/lib/coderay/encoders/term.rb +137 -0
  26. data/lib/coderay/encoders/text.rb +32 -0
  27. data/lib/coderay/encoders/token_class_filter.rb +84 -0
  28. data/lib/coderay/encoders/xml.rb +71 -0
  29. data/lib/coderay/encoders/yaml.rb +22 -0
  30. data/lib/coderay/for_redcloth.rb +85 -0
  31. data/lib/coderay/helpers/file_type.rb +240 -0
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +349 -0
  34. data/lib/coderay/helpers/word_list.rb +138 -0
  35. data/lib/coderay/scanner.rb +284 -0
  36. data/lib/coderay/scanners/_map.rb +23 -0
  37. data/lib/coderay/scanners/c.rb +203 -0
  38. data/lib/coderay/scanners/cpp.rb +228 -0
  39. data/lib/coderay/scanners/css.rb +210 -0
  40. data/lib/coderay/scanners/debug.rb +62 -0
  41. data/lib/coderay/scanners/delphi.rb +150 -0
  42. data/lib/coderay/scanners/diff.rb +105 -0
  43. data/lib/coderay/scanners/groovy.rb +263 -0
  44. data/lib/coderay/scanners/html.rb +182 -0
  45. data/lib/coderay/scanners/java.rb +176 -0
  46. data/lib/coderay/scanners/java/builtin_types.rb +419 -0
  47. data/lib/coderay/scanners/java_script.rb +224 -0
  48. data/lib/coderay/scanners/json.rb +112 -0
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +526 -0
  51. data/lib/coderay/scanners/plaintext.rb +21 -0
  52. data/lib/coderay/scanners/python.rb +285 -0
  53. data/lib/coderay/scanners/rhtml.rb +74 -0
  54. data/lib/coderay/scanners/ruby.rb +404 -0
  55. data/lib/coderay/scanners/ruby/patterns.rb +238 -0
  56. data/lib/coderay/scanners/scheme.rb +145 -0
  57. data/lib/coderay/scanners/sql.rb +162 -0
  58. data/lib/coderay/scanners/xml.rb +17 -0
  59. data/lib/coderay/scanners/yaml.rb +144 -0
  60. data/lib/coderay/style.rb +20 -0
  61. data/lib/coderay/styles/_map.rb +7 -0
  62. data/lib/coderay/styles/cycnus.rb +151 -0
  63. data/lib/coderay/styles/murphy.rb +132 -0
  64. data/lib/coderay/token_classes.rb +86 -0
  65. data/lib/coderay/tokens.rb +391 -0
  66. data/lib/term/ansicolor.rb +220 -0
  67. metadata +123 -0
@@ -0,0 +1,224 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class JavaScript < Scanner
5
+
6
+ include Streamable
7
+
8
+ register_for :java_script
9
+ file_extension 'js'
10
+
11
+ # The actual JavaScript keywords.
12
+ KEYWORDS = %w[
13
+ break case catch continue default delete do else
14
+ finally for function if in instanceof new
15
+ return switch throw try typeof var void while with
16
+ ]
17
+ PREDEFINED_CONSTANTS = %w[
18
+ false null true undefined
19
+ ]
20
+
21
+ MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
22
+
23
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
24
+ case delete in instanceof new return throw typeof with
25
+ ]
26
+
27
+ # Reserved for future use.
28
+ RESERVED_WORDS = %w[
29
+ abstract boolean byte char class debugger double enum export extends
30
+ final float goto implements import int interface long native package
31
+ private protected public short static super synchronized throws transient
32
+ volatile
33
+ ]
34
+
35
+ IDENT_KIND = WordList.new(:ident).
36
+ add(RESERVED_WORDS, :reserved).
37
+ add(PREDEFINED_CONSTANTS, :pre_constant).
38
+ add(MAGIC_VARIABLES, :local_variable).
39
+ add(KEYWORDS, :keyword)
40
+
41
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
42
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
43
+ REGEXP_ESCAPE = / [bBdDsSwW] /x
44
+ STRING_CONTENT_PATTERN = {
45
+ "'" => /[^\\']+/,
46
+ '"' => /[^\\"]+/,
47
+ '/' => /[^\\\/]+/,
48
+ }
49
+ KEY_CHECK_PATTERN = {
50
+ "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
51
+ '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
52
+ }
53
+
54
+ def scan_tokens tokens, options
55
+
56
+ state = :initial
57
+ string_delimiter = nil
58
+ value_expected = true
59
+ key_expected = false
60
+ function_expected = false
61
+
62
+ until eos?
63
+
64
+ kind = nil
65
+ match = nil
66
+
67
+ case state
68
+
69
+ when :initial
70
+
71
+ if match = scan(/ \s+ | \\\n /x)
72
+ value_expected = true if !value_expected && match.index(?\n)
73
+ tokens << [match, :space]
74
+ next
75
+
76
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
77
+ value_expected = true
78
+ kind = :comment
79
+
80
+ elsif check(/\.?\d/)
81
+ key_expected = value_expected = false
82
+ if scan(/0[xX][0-9A-Fa-f]+/)
83
+ kind = :hex
84
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
85
+ kind = :oct
86
+ elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
87
+ kind = :float
88
+ elsif scan(/\d+/)
89
+ kind = :integer
90
+ end
91
+
92
+ elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
93
+ # FIXME: scan over nested tags
94
+ xml_scanner.tokenize match
95
+ value_expected = false
96
+ next
97
+
98
+ elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
99
+ value_expected = true
100
+ last_operator = match[-1]
101
+ key_expected = (last_operator == ?{) || (last_operator == ?,)
102
+ function_expected = false
103
+ kind = :operator
104
+
105
+ elsif scan(/ [)\]}]+ /x)
106
+ function_expected = key_expected = value_expected = false
107
+ kind = :operator
108
+
109
+ elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
110
+ kind = IDENT_KIND[match]
111
+ value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
112
+ # TODO: labels
113
+ if kind == :ident
114
+ if match.index(?$) # $ allowed inside an identifier
115
+ kind = :predefined
116
+ elsif function_expected
117
+ kind = :function
118
+ elsif check(/\s*[=:]\s*function\b/)
119
+ kind = :function
120
+ elsif key_expected && check(/\s*:/)
121
+ kind = :key
122
+ end
123
+ end
124
+ function_expected = (kind == :keyword) && (match == 'function')
125
+ key_expected = false
126
+
127
+ elsif match = scan(/["']/)
128
+ if key_expected && check(KEY_CHECK_PATTERN[match])
129
+ state = :key
130
+ else
131
+ state = :string
132
+ end
133
+ tokens << [:open, state]
134
+ string_delimiter = match
135
+ kind = :delimiter
136
+
137
+ elsif value_expected && (match = scan(/\/(?=\S)/))
138
+ tokens << [:open, :regexp]
139
+ state = :regexp
140
+ string_delimiter = '/'
141
+ kind = :delimiter
142
+
143
+ elsif scan(/ \/ /x)
144
+ value_expected = true
145
+ key_expected = false
146
+ kind = :operator
147
+
148
+ else
149
+ getch
150
+ kind = :error
151
+
152
+ end
153
+
154
+ when :string, :regexp, :key
155
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
156
+ kind = :content
157
+ elsif match = scan(/["'\/]/)
158
+ tokens << [match, :delimiter]
159
+ if state == :regexp
160
+ modifiers = scan(/[gim]+/)
161
+ tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
162
+ end
163
+ tokens << [:close, state]
164
+ string_delimiter = nil
165
+ key_expected = value_expected = false
166
+ state = :initial
167
+ next
168
+ elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
169
+ if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
170
+ kind = :content
171
+ else
172
+ kind = :char
173
+ end
174
+ elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
175
+ kind = :char
176
+ elsif scan(/\\./m)
177
+ kind = :content
178
+ elsif scan(/ \\ | $ /x)
179
+ tokens << [:close, state]
180
+ kind = :error
181
+ key_expected = value_expected = false
182
+ state = :initial
183
+ else
184
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
185
+ end
186
+
187
+ else
188
+ raise_inspect 'Unknown state', tokens
189
+
190
+ end
191
+
192
+ match ||= matched
193
+ if $DEBUG and not kind
194
+ raise_inspect 'Error token %p in line %d' %
195
+ [[match, kind], line], tokens
196
+ end
197
+ raise_inspect 'Empty token', tokens unless match
198
+
199
+ tokens << [match, kind]
200
+
201
+ end
202
+
203
+ if [:string, :regexp].include? state
204
+ tokens << [:close, state]
205
+ end
206
+
207
+ tokens
208
+ end
209
+
210
+ protected
211
+
212
+ def reset_instance
213
+ super
214
+ @xml_scanner.reset if defined? @xml_scanner
215
+ end
216
+
217
+ def xml_scanner
218
+ @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => true
219
+ end
220
+
221
+ end
222
+
223
+ end
224
+ end
@@ -0,0 +1,112 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class JSON < Scanner
5
+
6
+ include Streamable
7
+
8
+ register_for :json
9
+ file_extension 'json'
10
+
11
+ KINDS_NOT_LOC = [
12
+ :float, :char, :content, :delimiter,
13
+ :error, :integer, :operator, :value,
14
+ ]
15
+
16
+ CONSTANTS = %w( true false null )
17
+ IDENT_KIND = WordList.new(:key).add(CONSTANTS, :value)
18
+
19
+ ESCAPE = / [bfnrt\\"\/] /x
20
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} /x
21
+
22
+ def scan_tokens tokens, options
23
+
24
+ state = :initial
25
+ stack = []
26
+ string_delimiter = nil
27
+ key_expected = false
28
+
29
+ until eos?
30
+
31
+ kind = nil
32
+ match = nil
33
+
34
+ case state
35
+
36
+ when :initial
37
+ if match = scan(/ \s+ | \\\n /x)
38
+ tokens << [match, :space]
39
+ next
40
+ elsif match = scan(/ [:,\[{\]}] /x)
41
+ kind = :operator
42
+ case match
43
+ when '{' then stack << :object; key_expected = true
44
+ when '[' then stack << :array
45
+ when ':' then key_expected = false
46
+ when ',' then key_expected = true if stack.last == :object
47
+ when '}', ']' then stack.pop # no error recovery, but works for valid JSON
48
+ end
49
+ elsif match = scan(/ true | false | null /x)
50
+ kind = IDENT_KIND[match]
51
+ elsif match = scan(/-?(?:0|[1-9]\d*)/)
52
+ kind = :integer
53
+ if scan(/\.\d+(?:[eE][-+]?\d+)?|[eE][-+]?\d+/)
54
+ match << matched
55
+ kind = :float
56
+ end
57
+ elsif match = scan(/"/)
58
+ state = key_expected ? :key : :string
59
+ tokens << [:open, state]
60
+ kind = :delimiter
61
+ else
62
+ getch
63
+ kind = :error
64
+ end
65
+
66
+ when :string, :key
67
+ if scan(/[^\\"]+/)
68
+ kind = :content
69
+ elsif scan(/"/)
70
+ tokens << ['"', :delimiter]
71
+ tokens << [:close, state]
72
+ state = :initial
73
+ next
74
+ elsif scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
75
+ kind = :char
76
+ elsif scan(/\\./m)
77
+ kind = :content
78
+ elsif scan(/ \\ | $ /x)
79
+ tokens << [:close, :delimiter]
80
+ kind = :error
81
+ state = :initial
82
+ else
83
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
84
+ end
85
+
86
+ else
87
+ raise_inspect 'Unknown state', tokens
88
+
89
+ end
90
+
91
+ match ||= matched
92
+ if $DEBUG and not kind
93
+ raise_inspect 'Error token %p in line %d' %
94
+ [[match, kind], line], tokens
95
+ end
96
+ raise_inspect 'Empty token', tokens unless match
97
+
98
+ tokens << [match, kind]
99
+
100
+ end
101
+
102
+ if [:string, :key].include? state
103
+ tokens << [:close, state]
104
+ end
105
+
106
+ tokens
107
+ end
108
+
109
+ end
110
+
111
+ end
112
+ end
@@ -0,0 +1,136 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ load :html
5
+ load :ruby
6
+
7
+ # Nitro XHTML Scanner
8
+ class NitroXHTML < Scanner
9
+
10
+ include Streamable
11
+ register_for :nitro_xhtml
12
+ file_extension :xhtml
13
+ title 'Nitro XHTML'
14
+
15
+ KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
16
+
17
+ NITRO_RUBY_BLOCK = /
18
+ <\?r
19
+ (?>
20
+ [^\?]*
21
+ (?> \?(?!>) [^\?]* )*
22
+ )
23
+ (?: \?> )?
24
+ |
25
+ <ruby>
26
+ (?>
27
+ [^<]*
28
+ (?> <(?!\/ruby>) [^<]* )*
29
+ )
30
+ (?: <\/ruby> )?
31
+ |
32
+ <%
33
+ (?>
34
+ [^%]*
35
+ (?> %(?!>) [^%]* )*
36
+ )
37
+ (?: %> )?
38
+ /mx
39
+
40
+ NITRO_VALUE_BLOCK = /
41
+ \#
42
+ (?:
43
+ \{
44
+ [^{}]*
45
+ (?>
46
+ \{ [^}]* \}
47
+ (?> [^{}]* )
48
+ )*
49
+ \}?
50
+ | \| [^|]* \|?
51
+ | \( [^)]* \)?
52
+ | \[ [^\]]* \]?
53
+ | \\ [^\\]* \\?
54
+ )
55
+ /x
56
+
57
+ NITRO_ENTITY = /
58
+ % (?: \#\d+ | \w+ ) ;
59
+ /
60
+
61
+ START_OF_RUBY = /
62
+ (?=[<\#%])
63
+ < (?: \?r | % | ruby> )
64
+ | \# [{(|]
65
+ | % (?: \#\d+ | \w+ ) ;
66
+ /x
67
+
68
+ CLOSING_PAREN = Hash.new do |h, p|
69
+ h[p] = p
70
+ end.update( {
71
+ '(' => ')',
72
+ '[' => ']',
73
+ '{' => '}',
74
+ } )
75
+
76
+ private
77
+
78
+ def setup
79
+ @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
80
+ @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
81
+ end
82
+
83
+ def reset_instance
84
+ super
85
+ @html_scanner.reset
86
+ end
87
+
88
+ def scan_tokens tokens, options
89
+
90
+ until eos?
91
+
92
+ if (match = scan_until(/(?=#{START_OF_RUBY})/o) || scan_until(/\z/)) and not match.empty?
93
+ @html_scanner.tokenize match
94
+
95
+ elsif match = scan(/#{NITRO_VALUE_BLOCK}/o)
96
+ start_tag = match[0,2]
97
+ delimiter = CLOSING_PAREN[start_tag[1,1]]
98
+ end_tag = match[-1,1] == delimiter ? delimiter : ''
99
+ tokens << [:open, :inline]
100
+ tokens << [start_tag, :inline_delimiter]
101
+ code = match[start_tag.size .. -1 - end_tag.size]
102
+ @ruby_scanner.tokenize code
103
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
104
+ tokens << [:close, :inline]
105
+
106
+ elsif match = scan(/#{NITRO_RUBY_BLOCK}/o)
107
+ start_tag = '<?r'
108
+ end_tag = match[-2,2] == '?>' ? '?>' : ''
109
+ tokens << [:open, :inline]
110
+ tokens << [start_tag, :inline_delimiter]
111
+ code = match[start_tag.size .. -(end_tag.size)-1]
112
+ @ruby_scanner.tokenize code
113
+ tokens << [end_tag, :inline_delimiter] unless end_tag.empty?
114
+ tokens << [:close, :inline]
115
+
116
+ elsif entity = scan(/#{NITRO_ENTITY}/o)
117
+ tokens << [entity, :entity]
118
+
119
+ elsif scan(/%/)
120
+ tokens << [matched, :error]
121
+
122
+ else
123
+ raise_inspect 'else-case reached!', tokens
124
+
125
+ end
126
+
127
+ end
128
+
129
+ tokens
130
+
131
+ end
132
+
133
+ end
134
+
135
+ end
136
+ end