rubylexer 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. data/History.txt +90 -0
  2. data/Manifest.txt +54 -3
  3. data/README.txt +4 -7
  4. data/Rakefile +3 -2
  5. data/lib/rubylexer.rb +856 -323
  6. data/lib/rubylexer/0.7.0.rb +11 -2
  7. data/lib/rubylexer/0.7.1.rb +2 -0
  8. data/lib/rubylexer/charhandler.rb +4 -4
  9. data/lib/rubylexer/context.rb +86 -9
  10. data/lib/rubylexer/rulexer.rb +455 -101
  11. data/lib/rubylexer/token.rb +166 -43
  12. data/lib/rubylexer/tokenprinter.rb +16 -8
  13. data/lib/rubylexer/version.rb +1 -1
  14. data/rubylexer.vpj +98 -0
  15. data/test/code/all_the_gems.rb +33 -0
  16. data/test/code/all_the_raas.rb +226 -0
  17. data/test/code/all_the_rubies.rb +2 -0
  18. data/test/code/deletewarns.rb +19 -1
  19. data/test/code/dumptokens.rb +39 -8
  20. data/test/code/errscan +2 -0
  21. data/test/code/isolate_error.rb +72 -0
  22. data/test/code/lexloop +14 -0
  23. data/test/code/locatetest.rb +150 -8
  24. data/test/code/regression.rb +109 -0
  25. data/test/code/rubylexervsruby.rb +53 -15
  26. data/test/code/strgen.rb +138 -0
  27. data/test/code/tarball.rb +144 -0
  28. data/test/code/testcases.rb +11 -0
  29. data/test/code/tokentest.rb +115 -24
  30. data/test/data/__eof2.rb +1 -0
  31. data/test/data/__eof5.rb +2 -0
  32. data/test/data/__eof6.rb +2 -0
  33. data/test/data/cvtesc.rb +17 -0
  34. data/test/data/g.rb +6 -0
  35. data/test/data/hd0.rb +3 -0
  36. data/test/data/hdateof.rb +2 -0
  37. data/test/data/hdempty.rb +3 -0
  38. data/test/data/hdr.rb +9 -0
  39. data/test/data/hdr_dos.rb +13 -0
  40. data/test/data/hdr_dos2.rb +18 -0
  41. data/test/data/heart.rb +2 -0
  42. data/test/data/here_escnl.rb +25 -0
  43. data/test/data/here_escnl_dos.rb +20 -0
  44. data/test/data/here_squote.rb +3 -0
  45. data/test/data/heremonsters.rb +140 -0
  46. data/test/data/heremonsters.rb.broken +68 -0
  47. data/test/data/heremonsters.rb.broken.save +68 -0
  48. data/test/data/heremonsters_dos.rb +140 -0
  49. data/test/data/heremonsters_dos.rb.broken +68 -0
  50. data/test/data/illegal_oneliners.rb +1 -0
  51. data/test/data/illegal_stanzas.rb +0 -0
  52. data/test/data/make_ws_strdelim.rb +22 -0
  53. data/test/data/maven2_builer_test.rb +82 -0
  54. data/test/data/migration.rb +8944 -0
  55. data/test/data/modl.rb +6 -0
  56. data/test/data/modl_dos.rb +7 -0
  57. data/test/data/modl_fails.rb +10 -0
  58. data/test/data/multilinestring.rb +6 -0
  59. data/test/data/oneliners.rb +555 -0
  60. data/test/data/p-op.rb +2 -0
  61. data/test/data/p.rb +3 -1710
  62. data/test/data/s.rb +90 -21
  63. data/test/data/simple.rb +1 -0
  64. data/test/data/simple_dos.rb +1 -0
  65. data/test/data/stanzas.rb +1194 -0
  66. data/test/data/strdelim_crlf.rb +6 -0
  67. data/test/data/stuff.rb +6 -0
  68. data/test/data/stuff2.rb +5 -0
  69. data/test/data/stuff3.rb +6 -0
  70. data/test/data/stuff4.rb +6 -0
  71. data/test/data/tkweird.rb +20 -0
  72. data/test/data/unending_stuff.rb +5 -0
  73. data/test/data/whatnot.rb +8 -0
  74. data/test/data/ws_strdelim.rb +0 -0
  75. data/test/test.sh +239 -0
  76. data/testing.txt +39 -50
  77. metadata +110 -12
  78. data/test/code/dl_all_gems.rb +0 -43
  79. data/test/code/unpack_all_gems.rb +0 -15
  80. data/test/data/gemlist.txt +0 -280
data/test/data/modl.rb ADDED
@@ -0,0 +1,6 @@
1
+ <<Z
2
+
3
+ #@m
4
+
5
+ Z
6
+
@@ -0,0 +1,7 @@
1
+ <<ENDModelCode
2
+
3
+ class #{@modelClassName}
4
+ end
5
+
6
+ ENDModelCode
7
+
@@ -0,0 +1,10 @@
1
+ modelCode = <<ENDModelCode
2
+
3
+ class #{@modelClassName} < ActiveRecord::Base
4
+ def self.listAll
5
+ find_by_sql("{'0'.CT.''}")
6
+ end
7
+ end
8
+
9
+ ENDModelCode
10
+
@@ -0,0 +1,6 @@
1
+ p "123
2
+ 456
3
+ 789\nabc
4
+ def\nghi
5
+ "
6
+
@@ -0,0 +1,555 @@
1
+ def x; yield end #this must be first!!!!
2
+ #the purpose of x{...} is to prevent the enclosed code from
3
+ #modifying the list of known local variables. it may be omitted
4
+ #in cases where it is known that no local vars are defined.
5
+
6
+ p a rescue b
7
+ p //e
8
+ p //u
9
+ p //n
10
+ p //s
11
+ p 0o
12
+
13
+ #hash, not block
14
+ def a(b) {} end
15
+ def a.b(c) {} end
16
+
17
+ def a.b i; end
18
+ def b i; end
19
+ #uh-oh, implicit parens end before i not after
20
+
21
+ return {}.size
22
+ 1.return {}.size
23
+
24
+ break {}.size
25
+ 1.break {}.size
26
+
27
+ next {}.size
28
+ 1.next {}.size
29
+
30
+ raise {}.to_s+"****"
31
+ 1.raise {}.to_s+"****"
32
+
33
+ throw {}.to_s+"****"
34
+ 1.throw {}.to_s+"****"
35
+
36
+ P ::Class
37
+ x{q=1;def q.foo; end}
38
+ #q should be varnametoken, both times
39
+ p %(1)
40
+
41
+ p %\hah, backslash as string delimiter\
42
+ p %\hah, #{backslash} as string delimiter\
43
+ def foo(bar=5,tt=6) end
44
+ wwww,eeee=1,2
45
+ x{a.b,c.d=1,2}
46
+ x{|a.b,c.d|}
47
+
48
+ p % foo
49
+ p % foo
50
+
51
+ p(% foo )
52
+ p(% foo )
53
+
54
+ p eval "%\sfoo\s"
55
+
56
+ p eval "%\tfoo\t"
57
+ p eval "%\vfoo\v"
58
+ p eval "%\rfoo\r"
59
+ p eval "%\nfoo\n"
60
+ p eval "%\0foo\0"
61
+
62
+ p eval "%\r\nfoo\r\n"
63
+
64
+ #foo
65
+ p()
66
+ p
67
+ p(1,2)
68
+ #these 2 lines should tokenize identically
69
+ p (1,2)
70
+ #except for the extra space on this one
71
+
72
+
73
+
74
+ p File
75
+ #<<'abc123def'
76
+ def (is_a?(File::Stat)).foofarendle;end
77
+ p( {:rest=>99})
78
+ p %{{{{#{"}}}"}}}}}
79
+
80
+ p :"jim";
81
+ p :'jim';
82
+ p %s/jim/;
83
+ p %s"jim";
84
+ p %s'jim';
85
+ p %s`jim`;
86
+ p %s[jim];
87
+ p %s{jim};
88
+ p %s(jim);
89
+ p %s<jim>;
90
+ p %s jim ;
91
+ x{ for bob in [100] do p(bob %(22)) end }
92
+
93
+ p [1,2,3,]
94
+ p({1=>2,3=>4,})
95
+ x{ p aaa,bbb,ccc=1,2,3 }
96
+ proc{|a,b,c,| p a,b,c }.call(1,2,3)
97
+
98
+ p (Module.instance_methods - Object.instance_methods).sort
99
+ # an outer set of implicit parens is needed
100
+ p(Module.instance_methods - Object.instance_methods).sort
101
+ #no outer implicit parens
102
+ "foo".slice (1-2).nil?
103
+ #no outer implicit parens
104
+ p (1-2).nil?
105
+ #outer implicit parens needed
106
+ p(1-2).nil?
107
+ #no outer implicit parens needed
108
+ def self.z(*); end
109
+ def self.zz(*,&x); end
110
+ def self.zzz(&x); end
111
+
112
+ z z z z z z {}
113
+ z z z z z z do end
114
+
115
+ (/ 1/)
116
+ p(/ 1/)
117
+ false and( true ? f : g )
118
+ false and( f .. g )
119
+ false and( f ... g )
120
+ p 1&2
121
+ #should be OperatorToken, not KeywordToken
122
+ p 1|2
123
+ #ditto
124
+ p 1**2
125
+ p 1*2
126
+ p 1%(2)
127
+ p 1^2
128
+ p 1+2
129
+ p 1-2
130
+ p 1/2
131
+
132
+ p 1?2:3
133
+ #keyword
134
+
135
+ p 1 ?2:3
136
+ #keyword
137
+
138
+ p 1? 2:3
139
+ #keyword
140
+
141
+ p 1?2 :3
142
+ #keyword
143
+
144
+ p 1?2: 3
145
+ #keyword
146
+
147
+ p 1 ? 2:3
148
+ #keyword
149
+
150
+ p 1 ?2 :3
151
+ #keyword
152
+
153
+ p 1 ?2: 3
154
+ #keyword
155
+
156
+ p 1? 2 :3
157
+ #keyword
158
+
159
+ p 1? 2: 3
160
+ #keyword
161
+
162
+ p 1?2 : 3
163
+ #keyword
164
+
165
+ p 1 ? 2 : 3
166
+ #keyword
167
+ p 1==2
168
+ p 1===2
169
+ p 1[2]
170
+ #keyword
171
+ p 1;2
172
+ #keyword
173
+ p 1,2
174
+ #keyword
175
+ p 1.2
176
+ p 1.class
177
+ #keyword
178
+ p 1..2
179
+ #keyword
180
+ p 1...2
181
+ #keyword
182
+ p 1<2
183
+ p 1>2
184
+ p 1<=2
185
+ p 1>=2
186
+ p 1<<2
187
+
188
+ p 1>>2
189
+ p 1!=2
190
+ #this is keyword
191
+ p 1=~2
192
+ p 1!~2
193
+ #this is keyword
194
+ p 1&&2
195
+ #...
196
+ p 1||2
197
+ #...
198
+ p 1<=>2
199
+
200
+ define_method(:foo, &proc{:bar})
201
+ define_method :foo, &proc{:bar}
202
+ define_method(:foo) &proc{:bar}
203
+ define_method :foo &proc{:bar}
204
+ p :k, *nil
205
+ p :k, *nil, &nil
206
+ p *nil
207
+ p *nil, &nil
208
+ p p :c, :d
209
+ def r;4 end; r=r.nil?
210
+ p ?e.+?y
211
+ p ?e.+ ?y
212
+ p ?e.-?y
213
+ p ?e.*?y
214
+ p ?e./?y
215
+ p ?e.<<?y
216
+ p ?e.%?y
217
+ p ?e.**?y
218
+ p ?e.&?y
219
+
220
+ p 0.9
221
+ p 1.45000000000000000000000000000000000000001
222
+ p 9.999999999999999999999999999999999999999999
223
+ p 9.999999999999999999999999999999999999999999e999999999999999999999999999
224
+ p 0b0100011001
225
+ p 0o123456701
226
+ p 0123456701
227
+
228
+ p 0x123456789abcdefABCDEF01
229
+
230
+ p "Hi, my name is #{"Slim #{(4)>2?"Whitman":"Shady"} "}."
231
+ p "Hi, my name is #{"Slim #{(4)<2?"Whitman":"Shady"} "}."
232
+
233
+ p(String *Class)
234
+
235
+ def String.*(right) [self,right] end
236
+ def String.<<(right) [self,:<<,right] end
237
+ def String./(right) [self,:/,right] end
238
+ def String.[](right) [self,:[],right] end
239
+ def Class.-@; [:-@, self] end
240
+ p(String::Class)
241
+ p(String:: Class)
242
+ p(String ::Class)
243
+ p(String :: Class)
244
+ p(String/Class)
245
+ p(String/ Class)
246
+ p(String /Class/)
247
+ p(String / Class)
248
+ #borken
249
+ p(String[Class])
250
+ p(String[ Class])
251
+ p(String [Class])
252
+ p(String [ Class])
253
+ p(String*Class)
254
+ p(String* Class)
255
+ p(String *Class)
256
+ p(String * Class)
257
+ undef :*,<<,/,[]
258
+
259
+ p(false::to_s)
260
+ p(false ::to_s)
261
+ p(false:: to_s)
262
+ p(false :: to_s)
263
+
264
+ alias p? p
265
+ alias P? p
266
+ alias [] p
267
+ alias <=> p
268
+
269
+ p:p8
270
+ false ? p: p8
271
+ p :p8
272
+ false ? p : p8
273
+
274
+ #false ? q:p8
275
+ false ? q: p8
276
+ #false ? q :p8
277
+ false ? q : p8
278
+
279
+ #false ? Q:p8
280
+ #gives ruby indigestion
281
+ false ? Q: p8
282
+ #false ? Q :p8
283
+ #gives ruby indigestion
284
+ false ? Q : p8
285
+
286
+ p?:p8
287
+ false ? p?: p8
288
+ p? :p8
289
+ false ? p? : p8
290
+
291
+ p??1
292
+ p? ?1
293
+ p(p?? 1 : 2)
294
+ p(p? ? 1 : 2)
295
+
296
+ P?:p8
297
+ false ? P?: p8
298
+ P? :p8
299
+ false ? P? : p8
300
+
301
+
302
+ P??1
303
+ P? ?1
304
+ p(P?? 1 : 2)
305
+ p(P? ? 1 : 2)
306
+
307
+ self.[]:p8
308
+ false ? self.[]: p8
309
+ self.[] :p8
310
+ false ? self.[] : p8
311
+
312
+ self.<=>:p8
313
+ false ? self.<=>: p8
314
+ self.<=> :p8
315
+ false ? self.<=> : p8
316
+
317
+ self <=>:p8
318
+ #false ? self <=>: p8
319
+ #gives ruby indigestion
320
+ self <=> :p8
321
+ #false ? self <=> : p8
322
+ #gives ruby indigestion
323
+
324
+ x{ mix=nil; p / 5/mix }
325
+
326
+ p :`
327
+
328
+ p{}
329
+ p {}
330
+
331
+ def nil.+(x) ~x end
332
+ def nil.[](*x) [x] end
333
+ p( p + 5 )
334
+ p( p +5 )
335
+ p( p+5 )
336
+ p( p[] )
337
+ p( p [] )
338
+ p( p [ ] )
339
+ class NilClass; undef +,[] end
340
+
341
+ #values
342
+ p Foou.new.[] -9
343
+ p Foou.new.[] +9
344
+ p Foou.new.[]!false
345
+ p Foou.new.[] !false
346
+ p Foou.new.[]~9
347
+ p Foou.new.[] ~9
348
+ p Foou.new.[] %(9)
349
+ p Foou.new.[] /9/
350
+ p Foou.new.[]$9
351
+ p Foou.new.[]a0
352
+ p Foou.new.[] $9
353
+ p Foou.new.[] a0
354
+
355
+ #ops
356
+ p Foou.new.[]-9
357
+ p Foou.new.[]+9
358
+ p Foou.new.[]<<9
359
+ p Foou.new.[]%9
360
+ p Foou.new.[]/9
361
+
362
+ #lambdas (ops)
363
+ p Foou.new.[]{9}
364
+ p Foou.new.[] {9}
365
+
366
+ if p then p end
367
+
368
+ p({:foo=>:bar})
369
+ #why does this work? i'd think that ':foo=' would be 1 token
370
+
371
+ p EMPTY = 0
372
+ p BLACK = 1
373
+ p WHITE = - BLACK
374
+ p~4
375
+ p:f
376
+ p(~4){}
377
+ p(:f){}
378
+ p Array("foo\nbar")
379
+
380
+
381
+
382
+ p +(4)
383
+ p -(4)
384
+
385
+ p :'\\'
386
+
387
+ Foop.bar 1,2
388
+ Foop::bar 3,4
389
+
390
+
391
+ p %s{symbol}
392
+ p :$1
393
+ p :$98349576875974523789734582394578
394
+ p( %r{\/$})
395
+ p( %r~<!include:([\/\w\.\-]+)>~m)
396
+
397
+ p "#$a #@b #@@c #{$a+@b+@@c}"
398
+ p "\#$a \#@b \#@@c \#{$a+@b+@@c}"
399
+ p '#$a #@b #@@c #{$a+@b+@@c}'
400
+ p '\#$a \#@b \#@@c \#{$a+@b+@@c}'
401
+ p %w"#$a #@b #@@c #{$a+@b+@@c}"
402
+ p %w"\#$a \#@b \#@@c \#{$a+@b+@@c}"
403
+ p %W"#$a #@b #@@c #{$a+@b+@@c}"
404
+ p %W"\#$a \#@b \#@@c \#{$a+@b+@@c}"
405
+ p %Q[#$a #@b #@@c #{$a+@b+@@c}]
406
+ p %Q[\#$a \#@b \#@@c \#{$a+@b+@@c}]
407
+ p `echo #$a #@b #@@c #{$a+@b+@@c}`
408
+ p `echo \#$a \#@b \#@@c \#{$a+@b+@@c}`
409
+ p(/#$a #@b #@@c #{$a+@b+@@c}/)
410
+ #p(/\#$a \#@b \#@@c \#{$a+@b+@@c}/) #moved to w.rb
411
+
412
+ x{ compile_body=outvar='foob'}
413
+ p "#{}"
414
+ p "#(1)"
415
+ def intialize(resolvers=[Hosts.new, DNS.new]) end
416
+ def environment(env = File.basename($0, '.*')) end
417
+
418
+ def ==(o) 444 end
419
+ def String.ffff4() self.to_s+"ffff" end
420
+ p *[]
421
+ #abc123def
422
+
423
+
424
+ p(1.+1)
425
+ p pppp
426
+ p "\""
427
+ p(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
428
+ p(:%)
429
+
430
+ x{ p( { :class => class_=0}) }
431
+ x{ p cls_name = {}[:class] }
432
+
433
+ p foo
434
+ p "#{$!.class}"
435
+ p :p
436
+ p(:p)
437
+ p(:"[]")
438
+ p :"[]"
439
+ p("\\")
440
+ p(/\\/)
441
+ p(/[\\]/)
442
+ p 0x80
443
+ p ?p
444
+ p 0.1
445
+ p 0.8
446
+ p 0.9
447
+ p(-1)
448
+ p %/p/
449
+ p %Q[<LI>]
450
+ i=99
451
+ p %Q[<LI><A HREF="#{i[3]}.html\##{i[4]}">#{i[0]+i[1]+(i[2])}</A>\n]
452
+ p(:side=>:top)
453
+ p %w[\\]
454
+ p %w[\]]
455
+ p :+
456
+ p 99 / 3
457
+ x{ a=99;b=3;p 1+(a / b) }
458
+ p %Q[\"]
459
+ p %Q[ some [nested] text]
460
+
461
+ p '\n'
462
+ p "\n"
463
+ p %w/\n/
464
+
465
+ p %W/\n/
466
+ p(/\n/)
467
+ p `\n`
468
+ p(%r[foo]i)
469
+ p ENV["AmritaCacheDir"]
470
+ x{ p f = 3.7517675036461267e+17 }
471
+ p $10
472
+ p $1001
473
+
474
+ def jd_to_wday(jd) (jd + 1) % 7 end
475
+ p jd_to_wday(98)
476
+ x{ p pre = $` }
477
+ p $-j=55
478
+
479
+ def empty() end
480
+ def printem2 a,b,c; p(a +77); p(b +77); p(c +77) end
481
+ def three() (1+2) end
482
+
483
+ def d;end
484
+ def d()end
485
+ def d(dd)end
486
+
487
+ p proc{||}
488
+ p "\v"
489
+ x{ c=0; while c == /[ \t\f\r\13]/; end }
490
+
491
+
492
+
493
+
494
+
495
+ def ~@; :foo end
496
+ undef ~@
497
+ alias ~@ non
498
+ alias non ~@
499
+ p :~@
500
+ a.~@
501
+ a::~@
502
+
503
+ JAVASCRIPTS.each { |script| @xml.script :type => 'text/javascript', :src => "/javascripts/#{script}.js" do end }
504
+ x{ script=0; @xml.script :type => text/javascript, :src => "/javascripts/#{script}.js" do end }
505
+ sources.each { |src| ant.src :path => src }
506
+
507
+ x{sn = m.sn :sen}
508
+ x{sn = m.sn %(sen)}
509
+ x{sn = m.sn ?s}
510
+ x{sn = m.sn /s/}
511
+ x{sn = m::sn :sen}
512
+
513
+ a b do end
514
+ a b.c do end
515
+ a b() do end
516
+ a b.c() do end
517
+ x{b=1;a b do end}
518
+ #b should be local var, both times
519
+ x{b=1;a=b do end}
520
+ #do applies to b
521
+ x{a=b do end}
522
+ #do applies to b
523
+ x{a b=c do end}
524
+ x{c=1;a b=c do end}
525
+ 6._?
526
+ 6._=7
527
+ 6._ =7
528
+ 6._= 7
529
+ 6._ = 7
530
+ E if defined? :E
531
+
532
+ defined? %/f/
533
+ defined? []
534
+ defined?({})
535
+ defined? ::A
536
+ next ::A
537
+ break ::A
538
+ return ::A
539
+ defined? ?A
540
+ defined? -1
541
+ defined? -1.0
542
+ defined? +1
543
+ defined? +1.0
544
+
545
+ defined? ~1
546
+ next ~1
547
+ return ~1
548
+ break ~1
549
+
550
+
551
+ def a.b; end rescue b0
552
+
553
+ def maybe(chance = 0.5)end
554
+ return rval / precision
555
+ 0e0