coderay 0.9.7.pre → 0.9.7

Sign up to get free protection for your applications and to get access to all the features.
data/lib/README CHANGED
@@ -18,7 +18,7 @@ And with line numbers.
18
18
  * is what everybody should have on their website
19
19
  * solves all your problems and makes the girls run after you
20
20
 
21
- Version: 0.9.6
21
+ Version: 0.9.7
22
22
  Author:: murphy (Kornelius Kalnbach)
23
23
  Contact:: murphy rubychan de
24
24
  Website:: coderay.rubychan.de[http://coderay.rubychan.de]
@@ -94,6 +94,7 @@ Please report errors in this documentation to <murphy rubychan de>.
94
94
  * Rob Aldred for the terminal encoder
95
95
  * Trans for pointing out $DEBUG dependencies
96
96
  * Flameeyes for finding that Term::ANSIColor was obsolete
97
+ * Etienne Massip for reporting a serious bug in JavaScript scanner
97
98
  * matz and all Ruby gods and gurus
98
99
  * The inventors of: the computer, the internet, the true color display, HTML &
99
100
  CSS, VIM, Ruby, pizza, microwaves, guitars, scouting, programming, anime,
@@ -0,0 +1,224 @@
1
+ module CodeRay
2
+ module Scanners
3
+
4
+ class JavaScript < Scanner
5
+
6
+ include Streamable
7
+
8
+ register_for :java_script
9
+ file_extension 'js'
10
+
11
+ # The actual JavaScript keywords.
12
+ KEYWORDS = %w[
13
+ break case catch continue default delete do else
14
+ finally for function if in instanceof new
15
+ return switch throw try typeof var void while with
16
+ ]
17
+ PREDEFINED_CONSTANTS = %w[
18
+ false null true undefined
19
+ ]
20
+
21
+ MAGIC_VARIABLES = %w[ this arguments ] # arguments was introduced in JavaScript 1.4
22
+
23
+ KEYWORDS_EXPECTING_VALUE = WordList.new.add %w[
24
+ case delete in instanceof new return throw typeof with
25
+ ]
26
+
27
+ # Reserved for future use.
28
+ RESERVED_WORDS = %w[
29
+ abstract boolean byte char class debugger double enum export extends
30
+ final float goto implements import int interface long native package
31
+ private protected public short static super synchronized throws transient
32
+ volatile
33
+ ]
34
+
35
+ IDENT_KIND = WordList.new(:ident).
36
+ add(RESERVED_WORDS, :reserved).
37
+ add(PREDEFINED_CONSTANTS, :pre_constant).
38
+ add(MAGIC_VARIABLES, :local_variable).
39
+ add(KEYWORDS, :keyword)
40
+
41
+ ESCAPE = / [bfnrtv\n\\'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} /x
42
+ UNICODE_ESCAPE = / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
43
+ REGEXP_ESCAPE = / [bBdDsSwW] /x
44
+ STRING_CONTENT_PATTERN = {
45
+ "'" => /[^\\']+/,
46
+ '"' => /[^\\"]+/,
47
+ '/' => /[^\\\/]+/,
48
+ }
49
+ KEY_CHECK_PATTERN = {
50
+ "'" => / [^\\']* (?: \\.? [^\\']* )* '? \s* : /x,
51
+ '"' => / [^\\"]* (?: \\.? [^\\"]* )* "? \s* : /x,
52
+ }
53
+
54
+ def scan_tokens tokens, options
55
+
56
+ state = :initial
57
+ string_delimiter = nil
58
+ value_expected = true
59
+ key_expected = false
60
+ function_expected = false
61
+
62
+ until eos?
63
+
64
+ kind = nil
65
+ match = nil
66
+
67
+ case state
68
+
69
+ when :initial
70
+
71
+ if match = scan(/ \s+ | \\\n /x)
72
+ value_expected = true if !value_expected && match.index(?\n)
73
+ tokens << [match, :space]
74
+ next
75
+
76
+ elsif scan(%r! // [^\n\\]* (?: \\. [^\n\\]* )* | /\* (?: .*? \*/ | .* ) !mx)
77
+ value_expected = true
78
+ kind = :comment
79
+
80
+ elsif check(/\.?\d/)
81
+ key_expected = value_expected = false
82
+ if scan(/0[xX][0-9A-Fa-f]+/)
83
+ kind = :hex
84
+ elsif scan(/(?>0[0-7]+)(?![89.eEfF])/)
85
+ kind = :oct
86
+ elsif scan(/\d+[fF]|\d*\.\d+(?:[eE][+-]?\d+)?[fF]?|\d+[eE][+-]?\d+[fF]?/)
87
+ kind = :float
88
+ elsif scan(/\d+/)
89
+ kind = :integer
90
+ end
91
+
92
+ elsif value_expected && match = scan(/<([[:alpha:]]\w*) (?: [^\/>]*\/> | .*?<\/\1>)/xim)
93
+ # FIXME: scan over nested tags
94
+ xml_scanner.tokenize match
95
+ value_expected = false
96
+ next
97
+
98
+ elsif match = scan(/ [-+*=<>?:;,!&^|(\[{~%]+ | \.(?!\d) /x)
99
+ value_expected = true
100
+ last_operator = match[-1]
101
+ key_expected = (last_operator == ?{) || (last_operator == ?,)
102
+ function_expected = false
103
+ kind = :operator
104
+
105
+ elsif scan(/ [)\]}]+ /x)
106
+ function_expected = key_expected = value_expected = false
107
+ kind = :operator
108
+
109
+ elsif match = scan(/ [$a-zA-Z_][A-Za-z_0-9$]* /x)
110
+ kind = IDENT_KIND[match]
111
+ value_expected = (kind == :keyword) && KEYWORDS_EXPECTING_VALUE[match]
112
+ # TODO: labels
113
+ if kind == :ident
114
+ if match.index(?$) # $ allowed inside an identifier
115
+ kind = :predefined
116
+ elsif function_expected
117
+ kind = :function
118
+ elsif check(/\s*[=:]\s*function\b/)
119
+ kind = :function
120
+ elsif key_expected && check(/\s*:/)
121
+ kind = :key
122
+ end
123
+ end
124
+ function_expected = (kind == :keyword) && (match == 'function')
125
+ key_expected = false
126
+
127
+ elsif match = scan(/["']/)
128
+ if key_expected && check(KEY_CHECK_PATTERN[match])
129
+ state = :key
130
+ else
131
+ state = :string
132
+ end
133
+ tokens << [:open, state]
134
+ string_delimiter = match
135
+ kind = :delimiter
136
+
137
+ elsif value_expected && (match = scan(/\/(?=\S)/))
138
+ tokens << [:open, :regexp]
139
+ state = :regexp
140
+ string_delimiter = '/'
141
+ kind = :delimiter
142
+
143
+ elsif scan(/ \/ /x)
144
+ value_expected = true
145
+ key_expected = false
146
+ kind = :operator
147
+
148
+ else
149
+ getch
150
+ kind = :error
151
+
152
+ end
153
+
154
+ when :string, :regexp, :key
155
+ if scan(STRING_CONTENT_PATTERN[string_delimiter])
156
+ kind = :content
157
+ elsif match = scan(/["'\/]/)
158
+ tokens << [match, :delimiter]
159
+ if state == :regexp
160
+ modifiers = scan(/[gim]+/)
161
+ tokens << [modifiers, :modifier] if modifiers && !modifiers.empty?
162
+ end
163
+ tokens << [:close, state]
164
+ string_delimiter = nil
165
+ key_expected = value_expected = false
166
+ state = :initial
167
+ next
168
+ elsif state != :regexp && (match = scan(/ \\ (?: #{ESCAPE} | #{UNICODE_ESCAPE} ) /mox))
169
+ if string_delimiter == "'" && !(match == "\\\\" || match == "\\'")
170
+ kind = :content
171
+ else
172
+ kind = :char
173
+ end
174
+ elsif state == :regexp && scan(/ \\ (?: #{ESCAPE} | #{REGEXP_ESCAPE} | #{UNICODE_ESCAPE} ) /mox)
175
+ kind = :char
176
+ elsif scan(/\\./m)
177
+ kind = :content
178
+ elsif scan(/ \\ | $ /x)
179
+ tokens << [:close, state]
180
+ kind = :error
181
+ key_expected = value_expected = false
182
+ state = :initial
183
+ else
184
+ raise_inspect "else case \" reached; %p not handled." % peek(1), tokens
185
+ end
186
+
187
+ else
188
+ raise_inspect 'Unknown state', tokens
189
+
190
+ end
191
+
192
+ match ||= matched
193
+ if $CODERAY_DEBUG and not kind
194
+ raise_inspect 'Error token %p in line %d' %
195
+ [[match, kind], line], tokens
196
+ end
197
+ raise_inspect 'Empty token', tokens unless match
198
+
199
+ tokens << [match, kind]
200
+
201
+ end
202
+
203
+ if [:string, :regexp].include? state
204
+ tokens << [:close, state]
205
+ end
206
+
207
+ tokens
208
+ end
209
+
210
+ protected
211
+
212
+ def reset_instance
213
+ super
214
+ @xml_scanner.reset if defined? @xml_scanner
215
+ end
216
+
217
+ def xml_scanner
218
+ @xml_scanner ||= CodeRay.scanner :xml, :tokens => @tokens, :keep_tokens => true, :keep_state => false
219
+ end
220
+
221
+ end
222
+
223
+ end
224
+ end
@@ -47,8 +47,8 @@ module Scanners
47
47
  '/' => /[^\\\/]+/,
48
48
  }
49
49
  KEY_CHECK_PATTERN = {
50
- "'" => / (?> [^\\']* (?: \\.? [^\\']* )* ) '? \s* : /x,
51
- '"' => / (?> [^\\"]* (?: \\.? [^\\"]* )* ) "? \s* : /x,
50
+ "'" => / (?> [^\\']* (?: \\. [^\\']* )* ) ' \s* : /mx,
51
+ '"' => / (?> [^\\"]* (?: \\. [^\\"]* )* ) " \s* : /mx,
52
52
  }
53
53
 
54
54
  def scan_tokens tokens, options
@@ -110,7 +110,7 @@ x
110
110
  9
111
111
  BasicTest
112
112
  i
113
- 249
113
+ 263
114
114
  5
115
115
  66
116
116
  99
@@ -359,6 +359,20 @@ i
359
359
  49
360
360
  3
361
361
  4
362
+ 15
363
+ 99
364
+ 7
365
+ 49
366
+ 7
367
+ 50
368
+ 65
369
+ 67
370
+ 49
371
+ 2
372
+ 0
373
+ 49
374
+ 3
375
+ 4
362
376
  11
363
377
  I
364
378
  a
@@ -370,7 +384,7 @@ I
370
384
  0
371
385
  n
372
386
  p
373
- 49
387
+ 51
374
388
  x
375
389
  12
376
390
  test_version
@@ -1807,8 +1821,74 @@ x
1807
1821
  /Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
1808
1822
  p
1809
1823
  0
1810
- p
1824
+ x
1825
+ 25
1826
+ test_scan_a_frozen_string
1827
+ M
1828
+ 1
1829
+ n
1830
+ n
1831
+ x
1832
+ 25
1833
+ test_scan_a_frozen_string
1834
+ i
1835
+ 12
1836
+ 45
1837
+ 0
1838
+ 1
1839
+ 45
1840
+ 2
1841
+ 3
1842
+ 7
1843
+ 4
1811
1844
  49
1845
+ 5
1846
+ 2
1847
+ 11
1848
+ I
1849
+ 3
1850
+ I
1851
+ 0
1852
+ I
1853
+ 0
1854
+ I
1855
+ 0
1856
+ n
1857
+ p
1858
+ 6
1859
+ x
1860
+ 7
1861
+ CodeRay
1862
+ n
1863
+ x
1864
+ 12
1865
+ RUBY_VERSION
1866
+ n
1867
+ x
1868
+ 4
1869
+ ruby
1870
+ x
1871
+ 4
1872
+ scan
1873
+ p
1874
+ 5
1875
+ I
1876
+ 0
1877
+ I
1878
+ 76
1879
+ I
1880
+ 0
1881
+ I
1882
+ 77
1883
+ I
1884
+ c
1885
+ x
1886
+ 55
1887
+ /Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
1888
+ p
1889
+ 0
1890
+ p
1891
+ 51
1812
1892
  I
1813
1893
  2
1814
1894
  I
@@ -1907,6 +1987,10 @@ I
1907
1987
  71
1908
1988
  I
1909
1989
  f9
1990
+ I
1991
+ 76
1992
+ I
1993
+ 107
1910
1994
  x
1911
1995
  55
1912
1996
  /Users/murphy/ruby/coderay-0.9/test/functional/basic.rb
@@ -1,5 +1,5 @@
1
1
  require 'test/unit'
2
- $: << 'lib'
2
+ $:.unshift 'lib'
3
3
  require 'coderay'
4
4
 
5
5
  begin
@@ -2,7 +2,7 @@ require 'test/unit'
2
2
 
3
3
  MYDIR = File.dirname(__FILE__)
4
4
 
5
- $: << 'lib'
5
+ $:.unshift 'lib'
6
6
  require 'coderay'
7
7
  puts "Running basic CodeRay #{CodeRay::VERSION} tests..."
8
8
 
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: coderay
3
3
  version: !ruby/object:Gem::Version
4
- hash: 961915920
5
- prerelease: true
4
+ hash: 53
5
+ prerelease: false
6
6
  segments:
7
7
  - 0
8
8
  - 9
9
9
  - 7
10
- - pre
11
- version: 0.9.7.pre
10
+ version: 0.9.7
12
11
  platform: ruby
13
12
  authors:
14
13
  - murphy
@@ -16,7 +15,7 @@ autorequire:
16
15
  bindir: bin
17
16
  cert_chain: []
18
17
 
19
- date: 2011-01-14 00:00:00 +01:00
18
+ date: 2011-01-15 00:00:00 +01:00
20
19
  default_executable:
21
20
  dependencies: []
22
21
 
@@ -74,6 +73,7 @@ files:
74
73
  - ./lib/coderay/scanners/html.rb
75
74
  - ./lib/coderay/scanners/java/builtin_types.rb
76
75
  - ./lib/coderay/scanners/java.rb
76
+ - ./lib/coderay/scanners/java_script-0.9.6.rb
77
77
  - ./lib/coderay/scanners/java_script.rb
78
78
  - ./lib/coderay/scanners/json.rb
79
79
  - ./lib/coderay/scanners/nitro_xhtml.rb
@@ -138,14 +138,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
138
138
  required_rubygems_version: !ruby/object:Gem::Requirement
139
139
  none: false
140
140
  requirements:
141
- - - ">"
141
+ - - ">="
142
142
  - !ruby/object:Gem::Version
143
- hash: 25
143
+ hash: 3
144
144
  segments:
145
- - 1
146
- - 3
147
- - 1
148
- version: 1.3.1
145
+ - 0
146
+ version: "0"
149
147
  requirements: []
150
148
 
151
149
  rubyforge_project: coderay