coderay-beta 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. data/FOLDERS +53 -0
  2. data/LICENSE +504 -0
  3. data/bin/coderay +82 -0
  4. data/bin/coderay_stylesheet +4 -0
  5. data/lib/README +129 -0
  6. data/lib/coderay.rb +320 -0
  7. data/lib/coderay/duo.rb +85 -0
  8. data/lib/coderay/encoder.rb +213 -0
  9. data/lib/coderay/encoders/_map.rb +11 -0
  10. data/lib/coderay/encoders/comment_filter.rb +43 -0
  11. data/lib/coderay/encoders/count.rb +21 -0
  12. data/lib/coderay/encoders/debug.rb +49 -0
  13. data/lib/coderay/encoders/div.rb +19 -0
  14. data/lib/coderay/encoders/filter.rb +75 -0
  15. data/lib/coderay/encoders/html.rb +305 -0
  16. data/lib/coderay/encoders/html/css.rb +70 -0
  17. data/lib/coderay/encoders/html/numerization.rb +133 -0
  18. data/lib/coderay/encoders/html/output.rb +206 -0
  19. data/lib/coderay/encoders/json.rb +69 -0
  20. data/lib/coderay/encoders/lines_of_code.rb +90 -0
  21. data/lib/coderay/encoders/null.rb +26 -0
  22. data/lib/coderay/encoders/page.rb +20 -0
  23. data/lib/coderay/encoders/span.rb +19 -0
  24. data/lib/coderay/encoders/statistic.rb +77 -0
  25. data/lib/coderay/encoders/term.rb +137 -0
  26. data/lib/coderay/encoders/text.rb +32 -0
  27. data/lib/coderay/encoders/token_class_filter.rb +84 -0
  28. data/lib/coderay/encoders/xml.rb +71 -0
  29. data/lib/coderay/encoders/yaml.rb +22 -0
  30. data/lib/coderay/for_redcloth.rb +85 -0
  31. data/lib/coderay/helpers/file_type.rb +240 -0
  32. data/lib/coderay/helpers/gzip_simple.rb +123 -0
  33. data/lib/coderay/helpers/plugin.rb +349 -0
  34. data/lib/coderay/helpers/word_list.rb +138 -0
  35. data/lib/coderay/scanner.rb +284 -0
  36. data/lib/coderay/scanners/_map.rb +23 -0
  37. data/lib/coderay/scanners/c.rb +203 -0
  38. data/lib/coderay/scanners/cpp.rb +228 -0
  39. data/lib/coderay/scanners/css.rb +210 -0
  40. data/lib/coderay/scanners/debug.rb +62 -0
  41. data/lib/coderay/scanners/delphi.rb +150 -0
  42. data/lib/coderay/scanners/diff.rb +105 -0
  43. data/lib/coderay/scanners/groovy.rb +263 -0
  44. data/lib/coderay/scanners/html.rb +182 -0
  45. data/lib/coderay/scanners/java.rb +176 -0
  46. data/lib/coderay/scanners/java/builtin_types.rb +419 -0
  47. data/lib/coderay/scanners/java_script.rb +224 -0
  48. data/lib/coderay/scanners/json.rb +112 -0
  49. data/lib/coderay/scanners/nitro_xhtml.rb +136 -0
  50. data/lib/coderay/scanners/php.rb +526 -0
  51. data/lib/coderay/scanners/plaintext.rb +21 -0
  52. data/lib/coderay/scanners/python.rb +285 -0
  53. data/lib/coderay/scanners/rhtml.rb +74 -0
  54. data/lib/coderay/scanners/ruby.rb +404 -0
  55. data/lib/coderay/scanners/ruby/patterns.rb +238 -0
  56. data/lib/coderay/scanners/scheme.rb +145 -0
  57. data/lib/coderay/scanners/sql.rb +162 -0
  58. data/lib/coderay/scanners/xml.rb +17 -0
  59. data/lib/coderay/scanners/yaml.rb +144 -0
  60. data/lib/coderay/style.rb +20 -0
  61. data/lib/coderay/styles/_map.rb +7 -0
  62. data/lib/coderay/styles/cycnus.rb +151 -0
  63. data/lib/coderay/styles/murphy.rb +132 -0
  64. data/lib/coderay/token_classes.rb +86 -0
  65. data/lib/coderay/tokens.rb +391 -0
  66. data/lib/term/ansicolor.rb +220 -0
  67. metadata +123 -0
@@ -0,0 +1,85 @@
1
+ module CodeRay
2
+
3
+ # = Duo
4
+ #
5
+ # A Duo is a convenient way to use CodeRay. You just create a Duo,
6
+ # giving it a lang (language of the input code) and a format (desired
7
+ # output format), and call Duo#highlight with the code.
8
+ #
9
+ # Duo makes it easy to re-use both scanner and encoder for a repetitive
10
+ # task. It also provides a very easy interface syntax:
11
+ #
12
+ # require 'coderay'
13
+ # CodeRay::Duo[:python, :div].highlight 'import this'
14
+ #
15
+ # Until you want to do uncommon things with CodeRay, I recommend to use
16
+ # this method, since it takes care of everything.
17
+ class Duo
18
+
19
+ attr_accessor :lang, :format, :options
20
+
21
+ # Create a new Duo, holding a lang and a format to highlight code.
22
+ #
23
+ # simple:
24
+ # CodeRay::Duo[:ruby, :page].highlight 'bla 42'
25
+ #
26
+ # streaming:
27
+ # CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
28
+ #
29
+ # with options:
30
+ # CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
31
+ #
32
+ # alternative syntax without options:
33
+ # CodeRay::Duo[:ruby => :statistic].encode 'class << self; end'
34
+ #
35
+ # alternative syntax with options:
36
+ # CodeRay::Duo[{ :ruby => :statistic }, :do => :something].encode 'abc'
37
+ #
38
+ # The options are forwarded to scanner and encoder
39
+ # (see CodeRay.get_scanner_options).
40
+ def initialize lang = nil, format = nil, options = {}
41
+ if format == nil and lang.is_a? Hash and lang.size == 1
42
+ @lang = lang.keys.first
43
+ @format = lang[@lang]
44
+ else
45
+ @lang = lang
46
+ @format = format
47
+ end
48
+ @options = options
49
+ end
50
+
51
+ class << self
52
+ # To allow calls like Duo[:ruby, :html].highlight.
53
+ alias [] new
54
+ end
55
+
56
+ # The scanner of the duo. Only created once.
57
+ def scanner
58
+ @scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
59
+ end
60
+
61
+ # The encoder of the duo. Only created once.
62
+ def encoder
63
+ @encoder ||= CodeRay.encoder @format, @options
64
+ end
65
+
66
+ # Tokenize and highlight the code using +scanner+ and +encoder+.
67
+ #
68
+ # If the :stream option is set, the Duo will go into streaming mode,
69
+ # saving memory for the cost of time.
70
+ def encode code, options = { :stream => false }
71
+ stream = options.delete :stream
72
+ options = @options.merge options
73
+ if stream
74
+ encoder.encode_stream(code, @lang, options)
75
+ else
76
+ scanner.code = code
77
+ encoder.encode_tokens(scanner.tokenize, options)
78
+ end
79
+ end
80
+ alias highlight encode
81
+
82
+ end
83
+
84
+ end
85
+
@@ -0,0 +1,213 @@
1
+ module CodeRay
2
+
3
+ # This module holds the Encoder class and its subclasses.
4
+ # For example, the HTML encoder is named CodeRay::Encoders::HTML
5
+ # can be found in coderay/encoders/html.
6
+ #
7
+ # Encoders also provides methods and constants for the register
8
+ # mechanism and the [] method that returns the Encoder class
9
+ # belonging to the given format.
10
+ module Encoders
11
+ extend PluginHost
12
+ plugin_path File.dirname(__FILE__), 'encoders'
13
+
14
+ # = Encoder
15
+ #
16
+ # The Encoder base class. Together with Scanner and
17
+ # Tokens, it forms the highlighting triad.
18
+ #
19
+ # Encoder instances take a Tokens object and do something with it.
20
+ #
21
+ # The most common Encoder is surely the HTML encoder
22
+ # (CodeRay::Encoders::HTML). It highlights the code in a colorful
23
+ # html page.
24
+ # If you want the highlighted code in a div or a span instead,
25
+ # use its subclasses Div and Span.
26
+ class Encoder
27
+ extend Plugin
28
+ plugin_host Encoders
29
+
30
+ attr_reader :token_stream
31
+
32
+ class << self
33
+
34
+ # Returns if the Encoder can be used in streaming mode.
35
+ def streamable?
36
+ is_a? Streamable
37
+ end
38
+
39
+ # If FILE_EXTENSION isn't defined, this method returns the
40
+ # downcase class name instead.
41
+ def const_missing sym
42
+ if sym == :FILE_EXTENSION
43
+ plugin_id
44
+ else
45
+ super
46
+ end
47
+ end
48
+
49
+ end
50
+
51
+ # Subclasses are to store their default options in this constant.
52
+ DEFAULT_OPTIONS = { :stream => false }
53
+
54
+ # The options you gave the Encoder at creating.
55
+ attr_accessor :options
56
+
57
+ # Creates a new Encoder.
58
+ # +options+ is saved and used for all encode operations, as long
59
+ # as you don't overwrite it there by passing additional options.
60
+ #
61
+ # Encoder objects provide three encode methods:
62
+ # - encode simply takes a +code+ string and a +lang+
63
+ # - encode_tokens expects a +tokens+ object instead
64
+ # - encode_stream is like encode, but uses streaming mode.
65
+ #
66
+ # Each method has an optional +options+ parameter. These are
67
+ # added to the options you passed at creation.
68
+ def initialize options = {}
69
+ @options = self.class::DEFAULT_OPTIONS.merge options
70
+ raise "I am only the basic Encoder class. I can't encode "\
71
+ "anything. :( Use my subclasses." if self.class == Encoder
72
+ end
73
+
74
+ # Encode a Tokens object.
75
+ def encode_tokens tokens, options = {}
76
+ options = @options.merge options
77
+ setup options
78
+ compile tokens, options
79
+ finish options
80
+ end
81
+
82
+ # Encode the given +code+ after tokenizing it using the Scanner
83
+ # for +lang+.
84
+ def encode code, lang, options = {}
85
+ options = @options.merge options
86
+ scanner_options = CodeRay.get_scanner_options(options)
87
+ tokens = CodeRay.scan code, lang, scanner_options
88
+ encode_tokens tokens, options
89
+ end
90
+
91
+ # You can use highlight instead of encode, if that seems
92
+ # more clear to you.
93
+ alias highlight encode
94
+
95
+ # Encode the given +code+ using the Scanner for +lang+ in
96
+ # streaming mode.
97
+ def encode_stream code, lang, options = {}
98
+ raise NotStreamableError, self unless kind_of? Streamable
99
+ options = @options.merge options
100
+ setup options
101
+ scanner_options = CodeRay.get_scanner_options options
102
+ @token_stream =
103
+ CodeRay.scan_stream code, lang, scanner_options, &self
104
+ finish options
105
+ end
106
+
107
+ # Behave like a proc. The token method is converted to a proc.
108
+ def to_proc
109
+ method(:token).to_proc
110
+ end
111
+
112
+ # Return the default file extension for outputs of this encoder.
113
+ def file_extension
114
+ self.class::FILE_EXTENSION
115
+ end
116
+
117
+ protected
118
+
119
+ # Called with merged options before encoding starts.
120
+ # Sets @out to an empty string.
121
+ #
122
+ # See the HTML Encoder for an example of option caching.
123
+ def setup options
124
+ @out = ''
125
+ end
126
+
127
+ # Called with +content+ and +kind+ of the currently scanned token.
128
+ # For simple scanners, it's enougth to implement this method.
129
+ #
130
+ # By default, it calls text_token or block_token, depending on
131
+ # whether +content+ is a String.
132
+ def token content, kind
133
+ encoded_token =
134
+ if content.is_a? ::String
135
+ text_token content, kind
136
+ elsif content.is_a? ::Symbol
137
+ block_token content, kind
138
+ else
139
+ raise 'Unknown token content type: %p' % [content]
140
+ end
141
+ append_encoded_token_to_output encoded_token
142
+ end
143
+
144
+ def append_encoded_token_to_output encoded_token
145
+ @out << encoded_token if encoded_token && defined?(@out) && @out
146
+ end
147
+
148
+ # Called for each text token ([text, kind]), where text is a String.
149
+ def text_token text, kind
150
+ end
151
+
152
+ # Called for each block (non-text) token ([action, kind]),
153
+ # where +action+ is a Symbol.
154
+ #
155
+ # Calls open_token, close_token, begin_line, and end_line according to
156
+ # the value of +action+.
157
+ def block_token action, kind
158
+ case action
159
+ when :open
160
+ open_token kind
161
+ when :close
162
+ close_token kind
163
+ when :begin_line
164
+ begin_line kind
165
+ when :end_line
166
+ end_line kind
167
+ else
168
+ raise 'unknown block action: %p' % action
169
+ end
170
+ end
171
+
172
+ # Called for each block token at the start of the block ([:open, kind]).
173
+ def open_token kind
174
+ end
175
+
176
+ # Called for each block token end of the block ([:close, kind]).
177
+ def close_token kind
178
+ end
179
+
180
+ # Called for each line token block at the start of the line ([:begin_line, kind]).
181
+ def begin_line kind
182
+ end
183
+
184
+ # Called for each line token block at the end of the line ([:end_line, kind]).
185
+ def end_line kind
186
+ end
187
+
188
+ # Called with merged options after encoding starts.
189
+ # The return value is the result of encoding, typically @out.
190
+ def finish options
191
+ @out
192
+ end
193
+
194
+ # Do the encoding.
195
+ #
196
+ # The already created +tokens+ object must be used; it can be a
197
+ # TokenStream or a Tokens object.
198
+ if RUBY_VERSION >= '1.9'
199
+ def compile tokens, options
200
+ for text, kind in tokens
201
+ token text, kind
202
+ end
203
+ end
204
+ else
205
+ def compile tokens, options
206
+ tokens.each(&self)
207
+ end
208
+ end
209
+
210
+ end
211
+
212
+ end
213
+ end
@@ -0,0 +1,11 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ map \
5
+ :loc => :lines_of_code,
6
+ :plain => :text,
7
+ :stats => :statistic,
8
+ :tex => :latex
9
+
10
+ end
11
+ end
@@ -0,0 +1,43 @@
1
+ ($:.unshift '../..'; require 'coderay') unless defined? CodeRay
2
+ module CodeRay
3
+ module Encoders
4
+
5
+ load :token_class_filter
6
+
7
+ class CommentFilter < TokenClassFilter
8
+
9
+ register_for :comment_filter
10
+
11
+ DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
12
+ :exclude => [:comment]
13
+
14
+ end
15
+
16
+ end
17
+ end
18
+
19
+ if $0 == __FILE__
20
+ $VERBOSE = true
21
+ $: << File.join(File.dirname(__FILE__), '..')
22
+ eval DATA.read, nil, $0, __LINE__ + 4
23
+ end
24
+
25
+ __END__
26
+ require 'test/unit'
27
+
28
+ class CommentFilterTest < Test::Unit::TestCase
29
+
30
+ def test_filtering_comments
31
+ tokens = CodeRay.scan <<-RUBY, :ruby
32
+ #!/usr/bin/env ruby
33
+ # a minimal Ruby program
34
+ puts "Hello world!"
35
+ RUBY
36
+ assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
37
+ #!/usr/bin/env ruby
38
+
39
+ puts "Hello world!"
40
+ RUBY_FILTERED
41
+ end
42
+
43
+ end
@@ -0,0 +1,21 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ class Count < Encoder
5
+
6
+ include Streamable
7
+ register_for :count
8
+
9
+ protected
10
+
11
+ def setup options
12
+ @out = 0
13
+ end
14
+
15
+ def token text, kind
16
+ @out += 1
17
+ end
18
+ end
19
+
20
+ end
21
+ end
@@ -0,0 +1,49 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ # = Debug Encoder
5
+ #
6
+ # Fast encoder producing simple debug output.
7
+ #
8
+ # It is readable and diff-able and is used for testing.
9
+ #
10
+ # You cannot fully restore the tokens information from the
11
+ # output, because consecutive :space tokens are merged.
12
+ # Use Tokens#dump for caching purposes.
13
+ class Debug < Encoder
14
+
15
+ include Streamable
16
+ register_for :debug
17
+
18
+ FILE_EXTENSION = 'raydebug'
19
+
20
+ protected
21
+ def text_token text, kind
22
+ if kind == :space
23
+ text
24
+ else
25
+ text = text.gsub(/[)\\]/, '\\\\\0') # escape ) and \
26
+ "#{kind}(#{text})"
27
+ end
28
+ end
29
+
30
+ def open_token kind
31
+ "#{kind}<"
32
+ end
33
+
34
+ def close_token kind
35
+ ">"
36
+ end
37
+
38
+ def begin_line kind
39
+ "#{kind}["
40
+ end
41
+
42
+ def end_line kind
43
+ "]"
44
+ end
45
+
46
+ end
47
+
48
+ end
49
+ end
@@ -0,0 +1,19 @@
1
+ module CodeRay
2
+ module Encoders
3
+
4
+ load :html
5
+
6
+ class Div < HTML
7
+
8
+ FILE_EXTENSION = 'div.html'
9
+
10
+ register_for :div
11
+
12
+ DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
13
+ :css => :style,
14
+ :wrap => :div
15
+
16
+ end
17
+
18
+ end
19
+ end