coderay 0.9.8 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/{lib/README → README_INDEX.rdoc} +10 -21
- data/Rakefile +6 -6
- data/bin/coderay +193 -64
- data/lib/coderay.rb +61 -105
- data/lib/coderay/duo.rb +17 -21
- data/lib/coderay/encoder.rb +100 -112
- data/lib/coderay/encoders/_map.rb +12 -7
- data/lib/coderay/encoders/comment_filter.rb +12 -30
- data/lib/coderay/encoders/count.rb +29 -11
- data/lib/coderay/encoders/debug.rb +32 -20
- data/lib/coderay/encoders/div.rb +13 -9
- data/lib/coderay/encoders/filter.rb +34 -51
- data/lib/coderay/encoders/html.rb +155 -161
- data/lib/coderay/encoders/html/css.rb +4 -9
- data/lib/coderay/encoders/html/numbering.rb +115 -0
- data/lib/coderay/encoders/html/output.rb +22 -70
- data/lib/coderay/encoders/json.rb +59 -45
- data/lib/coderay/encoders/lines_of_code.rb +12 -57
- data/lib/coderay/encoders/null.rb +6 -14
- data/lib/coderay/encoders/page.rb +13 -9
- data/lib/coderay/encoders/span.rb +13 -9
- data/lib/coderay/encoders/statistic.rb +58 -39
- data/lib/coderay/encoders/terminal.rb +179 -0
- data/lib/coderay/encoders/text.rb +31 -17
- data/lib/coderay/encoders/token_kind_filter.rb +111 -0
- data/lib/coderay/encoders/xml.rb +19 -18
- data/lib/coderay/encoders/yaml.rb +37 -9
- data/lib/coderay/for_redcloth.rb +4 -4
- data/lib/coderay/helpers/file_type.rb +127 -246
- data/lib/coderay/helpers/gzip.rb +41 -0
- data/lib/coderay/helpers/plugin.rb +241 -306
- data/lib/coderay/helpers/word_list.rb +65 -126
- data/lib/coderay/scanner.rb +173 -156
- data/lib/coderay/scanners/_map.rb +18 -17
- data/lib/coderay/scanners/c.rb +63 -77
- data/lib/coderay/scanners/clojure.rb +217 -0
- data/lib/coderay/scanners/cpp.rb +71 -84
- data/lib/coderay/scanners/css.rb +103 -120
- data/lib/coderay/scanners/debug.rb +47 -44
- data/lib/coderay/scanners/delphi.rb +70 -76
- data/lib/coderay/scanners/diff.rb +141 -50
- data/lib/coderay/scanners/erb.rb +81 -0
- data/lib/coderay/scanners/groovy.rb +104 -113
- data/lib/coderay/scanners/haml.rb +168 -0
- data/lib/coderay/scanners/html.rb +181 -110
- data/lib/coderay/scanners/java.rb +73 -75
- data/lib/coderay/scanners/java/builtin_types.rb +2 -0
- data/lib/coderay/scanners/java_script.rb +90 -101
- data/lib/coderay/scanners/json.rb +40 -53
- data/lib/coderay/scanners/php.rb +123 -147
- data/lib/coderay/scanners/python.rb +93 -91
- data/lib/coderay/scanners/raydebug.rb +66 -0
- data/lib/coderay/scanners/ruby.rb +343 -326
- data/lib/coderay/scanners/ruby/patterns.rb +40 -106
- data/lib/coderay/scanners/ruby/string_state.rb +71 -0
- data/lib/coderay/scanners/sql.rb +80 -66
- data/lib/coderay/scanners/text.rb +26 -0
- data/lib/coderay/scanners/xml.rb +1 -1
- data/lib/coderay/scanners/yaml.rb +74 -73
- data/lib/coderay/style.rb +10 -7
- data/lib/coderay/styles/_map.rb +3 -3
- data/lib/coderay/styles/alpha.rb +143 -0
- data/lib/coderay/token_kinds.rb +90 -0
- data/lib/coderay/tokens.rb +102 -277
- data/lib/coderay/tokens_proxy.rb +55 -0
- data/lib/coderay/version.rb +3 -0
- data/test/functional/basic.rb +200 -18
- data/test/functional/examples.rb +130 -0
- data/test/functional/for_redcloth.rb +15 -8
- data/test/functional/suite.rb +9 -6
- metadata +103 -123
- data/FOLDERS +0 -53
- data/bin/coderay_stylesheet +0 -4
- data/lib/coderay/encoders/html/numerization.rb +0 -133
- data/lib/coderay/encoders/term.rb +0 -158
- data/lib/coderay/encoders/token_class_filter.rb +0 -84
- data/lib/coderay/helpers/gzip_simple.rb +0 -123
- data/lib/coderay/scanners/nitro_xhtml.rb +0 -136
- data/lib/coderay/scanners/plaintext.rb +0 -20
- data/lib/coderay/scanners/rhtml.rb +0 -78
- data/lib/coderay/scanners/scheme.rb +0 -145
- data/lib/coderay/styles/cycnus.rb +0 -152
- data/lib/coderay/styles/murphy.rb +0 -134
- data/lib/coderay/token_classes.rb +0 -86
- data/test/functional/load_plugin_scanner.rb +0 -11
- data/test/functional/vhdl.rb +0 -126
- data/test/functional/word_list.rb +0 -79
data/lib/coderay/duo.rb
CHANGED
@@ -21,10 +21,7 @@ module CodeRay
|
|
21
21
|
# Create a new Duo, holding a lang and a format to highlight code.
|
22
22
|
#
|
23
23
|
# simple:
|
24
|
-
# CodeRay::Duo[:ruby, :
|
25
|
-
#
|
26
|
-
# streaming:
|
27
|
-
# CodeRay::Duo[:ruby, :page].highlight 'bar 23', :stream => true
|
24
|
+
# CodeRay::Duo[:ruby, :html].highlight 'bla 42'
|
28
25
|
#
|
29
26
|
# with options:
|
30
27
|
# CodeRay::Duo[:ruby, :html, :hint => :debug].highlight '????::??'
|
@@ -38,7 +35,7 @@ module CodeRay
|
|
38
35
|
# The options are forwarded to scanner and encoder
|
39
36
|
# (see CodeRay.get_scanner_options).
|
40
37
|
def initialize lang = nil, format = nil, options = {}
|
41
|
-
if format
|
38
|
+
if format.nil? && lang.is_a?(Hash) && lang.size == 1
|
42
39
|
@lang = lang.keys.first
|
43
40
|
@format = lang[@lang]
|
44
41
|
else
|
@@ -47,12 +44,12 @@ module CodeRay
|
|
47
44
|
end
|
48
45
|
@options = options
|
49
46
|
end
|
50
|
-
|
47
|
+
|
51
48
|
class << self
|
52
49
|
# To allow calls like Duo[:ruby, :html].highlight.
|
53
50
|
alias [] new
|
54
51
|
end
|
55
|
-
|
52
|
+
|
56
53
|
# The scanner of the duo. Only created once.
|
57
54
|
def scanner
|
58
55
|
@scanner ||= CodeRay.scanner @lang, CodeRay.get_scanner_options(@options)
|
@@ -64,22 +61,21 @@ module CodeRay
|
|
64
61
|
end
|
65
62
|
|
66
63
|
# Tokenize and highlight the code using +scanner+ and +encoder+.
|
67
|
-
|
68
|
-
# If the :stream option is set, the Duo will go into streaming mode,
|
69
|
-
# saving memory for the cost of time.
|
70
|
-
def encode code, options = { :stream => false }
|
71
|
-
stream = options.delete :stream
|
64
|
+
def encode code, options = {}
|
72
65
|
options = @options.merge options
|
73
|
-
|
74
|
-
encoder.encode_stream(code, @lang, options)
|
75
|
-
else
|
76
|
-
scanner.code = code
|
77
|
-
encoder.encode_tokens(scanner.tokenize, options)
|
78
|
-
end
|
66
|
+
encoder.encode(code, @lang, options)
|
79
67
|
end
|
80
68
|
alias highlight encode
|
81
|
-
|
69
|
+
|
70
|
+
# Allows to use Duo like a proc object:
|
71
|
+
#
|
72
|
+
# CodeRay::Duo[:python => :yaml].call(code)
|
73
|
+
#
|
74
|
+
# or, in Ruby 1.9 and later:
|
75
|
+
#
|
76
|
+
# CodeRay::Duo[:python => :yaml].(code)
|
77
|
+
alias call encode
|
78
|
+
|
82
79
|
end
|
83
|
-
|
80
|
+
|
84
81
|
end
|
85
|
-
|
data/lib/coderay/encoder.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
module CodeRay
|
2
|
-
|
2
|
+
|
3
3
|
# This module holds the Encoder class and its subclasses.
|
4
4
|
# For example, the HTML encoder is named CodeRay::Encoders::HTML
|
5
5
|
# can be found in coderay/encoders/html.
|
@@ -8,9 +8,10 @@ module CodeRay
|
|
8
8
|
# mechanism and the [] method that returns the Encoder class
|
9
9
|
# belonging to the given format.
|
10
10
|
module Encoders
|
11
|
+
|
11
12
|
extend PluginHost
|
12
13
|
plugin_path File.dirname(__FILE__), 'encoders'
|
13
|
-
|
14
|
+
|
14
15
|
# = Encoder
|
15
16
|
#
|
16
17
|
# The Encoder base class. Together with Scanner and
|
@@ -26,34 +27,32 @@ module CodeRay
|
|
26
27
|
class Encoder
|
27
28
|
extend Plugin
|
28
29
|
plugin_host Encoders
|
29
|
-
|
30
|
-
attr_reader :token_stream
|
31
|
-
|
30
|
+
|
32
31
|
class << self
|
33
|
-
|
34
|
-
# Returns if the Encoder can be used in streaming mode.
|
35
|
-
def streamable?
|
36
|
-
is_a? Streamable
|
37
|
-
end
|
38
|
-
|
32
|
+
|
39
33
|
# If FILE_EXTENSION isn't defined, this method returns the
|
40
34
|
# downcase class name instead.
|
41
35
|
def const_missing sym
|
42
36
|
if sym == :FILE_EXTENSION
|
43
|
-
plugin_id
|
37
|
+
(defined?(@plugin_id) && @plugin_id || name[/\w+$/].downcase).to_s
|
44
38
|
else
|
45
39
|
super
|
46
40
|
end
|
47
41
|
end
|
48
|
-
|
42
|
+
|
43
|
+
# The default file extension for output file of this encoder class.
|
44
|
+
def file_extension
|
45
|
+
self::FILE_EXTENSION
|
46
|
+
end
|
47
|
+
|
49
48
|
end
|
50
|
-
|
49
|
+
|
51
50
|
# Subclasses are to store their default options in this constant.
|
52
|
-
DEFAULT_OPTIONS = {
|
53
|
-
|
51
|
+
DEFAULT_OPTIONS = { }
|
52
|
+
|
54
53
|
# The options you gave the Encoder at creating.
|
55
|
-
attr_accessor :options
|
56
|
-
|
54
|
+
attr_accessor :options, :scanner
|
55
|
+
|
57
56
|
# Creates a new Encoder.
|
58
57
|
# +options+ is saved and used for all encode operations, as long
|
59
58
|
# as you don't overwrite it there by passing additional options.
|
@@ -61,153 +60,142 @@ module CodeRay
|
|
61
60
|
# Encoder objects provide three encode methods:
|
62
61
|
# - encode simply takes a +code+ string and a +lang+
|
63
62
|
# - encode_tokens expects a +tokens+ object instead
|
64
|
-
# - encode_stream is like encode, but uses streaming mode.
|
65
63
|
#
|
66
64
|
# Each method has an optional +options+ parameter. These are
|
67
65
|
# added to the options you passed at creation.
|
68
66
|
def initialize options = {}
|
69
67
|
@options = self.class::DEFAULT_OPTIONS.merge options
|
70
|
-
|
71
|
-
"anything. :( Use my subclasses." if self.class == Encoder
|
68
|
+
@@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = false
|
72
69
|
end
|
73
|
-
|
70
|
+
|
74
71
|
# Encode a Tokens object.
|
75
72
|
def encode_tokens tokens, options = {}
|
76
73
|
options = @options.merge options
|
74
|
+
@scanner = tokens.scanner if tokens.respond_to? :scanner
|
77
75
|
setup options
|
78
76
|
compile tokens, options
|
79
77
|
finish options
|
80
78
|
end
|
81
|
-
|
82
|
-
# Encode the given +code+
|
83
|
-
# for +lang+.
|
79
|
+
|
80
|
+
# Encode the given +code+ using the Scanner for +lang+.
|
84
81
|
def encode code, lang, options = {}
|
85
82
|
options = @options.merge options
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
@scanner = Scanners[lang].new code, CodeRay.get_scanner_options(options).update(:tokens => self)
|
84
|
+
setup options
|
85
|
+
@scanner.tokenize
|
86
|
+
finish options
|
89
87
|
end
|
90
|
-
|
88
|
+
|
91
89
|
# You can use highlight instead of encode, if that seems
|
92
90
|
# more clear to you.
|
93
91
|
alias highlight encode
|
94
|
-
|
95
|
-
#
|
96
|
-
# streaming mode.
|
97
|
-
def encode_stream code, lang, options = {}
|
98
|
-
raise NotStreamableError, self unless kind_of? Streamable
|
99
|
-
options = @options.merge options
|
100
|
-
setup options
|
101
|
-
scanner_options = CodeRay.get_scanner_options options
|
102
|
-
@token_stream =
|
103
|
-
CodeRay.scan_stream code, lang, scanner_options, &self
|
104
|
-
finish options
|
105
|
-
end
|
106
|
-
|
107
|
-
# Behave like a proc. The token method is converted to a proc.
|
108
|
-
def to_proc
|
109
|
-
method(:token).to_proc
|
110
|
-
end
|
111
|
-
|
112
|
-
# Return the default file extension for outputs of this encoder.
|
92
|
+
|
93
|
+
# The default file extension for this encoder.
|
113
94
|
def file_extension
|
114
|
-
self.class
|
95
|
+
self.class.file_extension
|
115
96
|
end
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
def setup options
|
124
|
-
@out = ''
|
97
|
+
|
98
|
+
def << token
|
99
|
+
unless @@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN
|
100
|
+
warn 'Using old Tokens#<< interface.'
|
101
|
+
@@CODERAY_TOKEN_INTERFACE_DEPRECATION_WARNING_GIVEN = true
|
102
|
+
end
|
103
|
+
self.token(*token)
|
125
104
|
end
|
126
|
-
|
105
|
+
|
127
106
|
# Called with +content+ and +kind+ of the currently scanned token.
|
128
107
|
# For simple scanners, it's enougth to implement this method.
|
129
108
|
#
|
130
|
-
# By default, it calls text_token
|
131
|
-
#
|
109
|
+
# By default, it calls text_token, begin_group, end_group, begin_line,
|
110
|
+
# or end_line, depending on the +content+.
|
132
111
|
def token content, kind
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
end
|
141
|
-
append_encoded_token_to_output encoded_token
|
142
|
-
end
|
143
|
-
|
144
|
-
def append_encoded_token_to_output encoded_token
|
145
|
-
@out << encoded_token if encoded_token && defined?(@out) && @out
|
146
|
-
end
|
147
|
-
|
148
|
-
# Called for each text token ([text, kind]), where text is a String.
|
149
|
-
def text_token text, kind
|
150
|
-
end
|
151
|
-
|
152
|
-
# Called for each block (non-text) token ([action, kind]),
|
153
|
-
# where +action+ is a Symbol.
|
154
|
-
#
|
155
|
-
# Calls open_token, close_token, begin_line, and end_line according to
|
156
|
-
# the value of +action+.
|
157
|
-
def block_token action, kind
|
158
|
-
case action
|
159
|
-
when :open
|
160
|
-
open_token kind
|
161
|
-
when :close
|
162
|
-
close_token kind
|
112
|
+
case content
|
113
|
+
when String
|
114
|
+
text_token content, kind
|
115
|
+
when :begin_group
|
116
|
+
begin_group kind
|
117
|
+
when :end_group
|
118
|
+
end_group kind
|
163
119
|
when :begin_line
|
164
120
|
begin_line kind
|
165
121
|
when :end_line
|
166
122
|
end_line kind
|
167
123
|
else
|
168
|
-
raise '
|
124
|
+
raise ArgumentError, 'Unknown token content type: %p, kind = %p' % [content, kind]
|
169
125
|
end
|
170
126
|
end
|
171
127
|
|
172
|
-
# Called for each
|
173
|
-
def
|
128
|
+
# Called for each text token ([text, kind]), where text is a String.
|
129
|
+
def text_token text, kind
|
130
|
+
@out << text
|
174
131
|
end
|
175
132
|
|
176
|
-
#
|
177
|
-
def
|
133
|
+
# Starts a token group with the given +kind+.
|
134
|
+
def begin_group kind
|
178
135
|
end
|
179
136
|
|
180
|
-
#
|
137
|
+
# Ends a token group with the given +kind+.
|
138
|
+
def end_group kind
|
139
|
+
end
|
140
|
+
|
141
|
+
# Starts a new line token group with the given +kind+.
|
181
142
|
def begin_line kind
|
182
143
|
end
|
183
144
|
|
184
|
-
#
|
145
|
+
# Ends a new line token group with the given +kind+.
|
185
146
|
def end_line kind
|
186
147
|
end
|
187
|
-
|
148
|
+
|
149
|
+
protected
|
150
|
+
|
151
|
+
# Called with merged options before encoding starts.
|
152
|
+
# Sets @out to an empty string.
|
153
|
+
#
|
154
|
+
# See the HTML Encoder for an example of option caching.
|
155
|
+
def setup options
|
156
|
+
@out = get_output(options)
|
157
|
+
end
|
158
|
+
|
159
|
+
def get_output options
|
160
|
+
options[:out] || ''
|
161
|
+
end
|
162
|
+
|
163
|
+
# Append data.to_s to the output. Returns the argument.
|
164
|
+
def output data
|
165
|
+
@out << data.to_s
|
166
|
+
data
|
167
|
+
end
|
168
|
+
|
188
169
|
# Called with merged options after encoding starts.
|
189
170
|
# The return value is the result of encoding, typically @out.
|
190
171
|
def finish options
|
191
172
|
@out
|
192
173
|
end
|
193
|
-
|
174
|
+
|
194
175
|
# Do the encoding.
|
195
176
|
#
|
196
|
-
# The already created +tokens+ object must be used; it
|
197
|
-
#
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
177
|
+
# The already created +tokens+ object must be used; it must be a
|
178
|
+
# Tokens object.
|
179
|
+
def compile tokens, options = {}
|
180
|
+
content = nil
|
181
|
+
for item in tokens
|
182
|
+
if item.is_a? Array
|
183
|
+
raise ArgumentError, 'Two-element array tokens are no longer supported.'
|
184
|
+
end
|
185
|
+
if content
|
186
|
+
token content, item
|
187
|
+
content = nil
|
188
|
+
else
|
189
|
+
content = item
|
202
190
|
end
|
203
191
|
end
|
204
|
-
|
205
|
-
def compile tokens, options
|
206
|
-
tokens.each(&self)
|
207
|
-
end
|
192
|
+
raise 'odd number list for Tokens' if content
|
208
193
|
end
|
209
|
-
|
194
|
+
|
195
|
+
alias tokens compile
|
196
|
+
public :tokens
|
197
|
+
|
210
198
|
end
|
211
|
-
|
199
|
+
|
212
200
|
end
|
213
201
|
end
|
@@ -1,12 +1,17 @@
|
|
1
1
|
module CodeRay
|
2
2
|
module Encoders
|
3
|
-
|
3
|
+
|
4
4
|
map \
|
5
|
-
:loc
|
6
|
-
:plain
|
7
|
-
:
|
8
|
-
:
|
9
|
-
:
|
10
|
-
|
5
|
+
:loc => :lines_of_code,
|
6
|
+
:plain => :text,
|
7
|
+
:plaintext => :text,
|
8
|
+
:remove_comments => :comment_filter,
|
9
|
+
:stats => :statistic,
|
10
|
+
:term => :terminal,
|
11
|
+
:tty => :terminal,
|
12
|
+
:yml => :yaml
|
13
|
+
|
14
|
+
# No default because Tokens#nonsense should raise NoMethodError.
|
15
|
+
|
11
16
|
end
|
12
17
|
end
|
@@ -1,43 +1,25 @@
|
|
1
|
-
($:.unshift '../..'; require 'coderay') unless defined? CodeRay
|
2
1
|
module CodeRay
|
3
2
|
module Encoders
|
4
3
|
|
5
|
-
load :
|
4
|
+
load :token_kind_filter
|
6
5
|
|
7
|
-
|
6
|
+
# A simple Filter that removes all tokens of the :comment kind.
|
7
|
+
#
|
8
|
+
# Alias: +remove_comments+
|
9
|
+
#
|
10
|
+
# Usage:
|
11
|
+
# CodeRay.scan('print # foo', :ruby).comment_filter.text
|
12
|
+
# #-> "print "
|
13
|
+
#
|
14
|
+
# See also: TokenKindFilter, LinesOfCode
|
15
|
+
class CommentFilter < TokenKindFilter
|
8
16
|
|
9
17
|
register_for :comment_filter
|
10
18
|
|
11
19
|
DEFAULT_OPTIONS = superclass::DEFAULT_OPTIONS.merge \
|
12
|
-
:exclude => [:comment]
|
20
|
+
:exclude => [:comment, :docstring]
|
13
21
|
|
14
22
|
end
|
15
23
|
|
16
24
|
end
|
17
25
|
end
|
18
|
-
|
19
|
-
if $0 == __FILE__
|
20
|
-
$VERBOSE = true
|
21
|
-
$: << File.join(File.dirname(__FILE__), '..')
|
22
|
-
eval DATA.read, nil, $0, __LINE__ + 4
|
23
|
-
end
|
24
|
-
|
25
|
-
__END__
|
26
|
-
require 'test/unit'
|
27
|
-
|
28
|
-
class CommentFilterTest < Test::Unit::TestCase
|
29
|
-
|
30
|
-
def test_filtering_comments
|
31
|
-
tokens = CodeRay.scan <<-RUBY, :ruby
|
32
|
-
#!/usr/bin/env ruby
|
33
|
-
# a minimal Ruby program
|
34
|
-
puts "Hello world!"
|
35
|
-
RUBY
|
36
|
-
assert_equal <<-RUBY_FILTERED, tokens.comment_filter.text
|
37
|
-
#!/usr/bin/env ruby
|
38
|
-
|
39
|
-
puts "Hello world!"
|
40
|
-
RUBY_FILTERED
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|