rexical 1.0.3 → 1.0.4

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of rexical might be problematic. Click here for more details.

@@ -1,3 +1,9 @@
1
+ === 1.0.4
2
+
3
+ * Bug fixes
4
+
5
+ * Generated tokenizer only tokenizes on pulls
6
+
1
7
  === 1.0.3
2
8
 
3
9
  * Bug fixes
@@ -8,7 +8,6 @@ Rakefile
8
8
  bin/rex
9
9
  lib/rexical.rb
10
10
  lib/rexical/generator.rb
11
- lib/rexical/info.rb
12
11
  lib/rexical/rexcmd.rb
13
12
  sample/a.cmd
14
13
  sample/b.cmd
data/Rakefile CHANGED
@@ -3,15 +3,12 @@
3
3
  require 'rubygems'
4
4
  require 'hoe'
5
5
 
6
- $LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
7
- require 'rexical'
8
-
9
- HOE = Hoe.new('rexical', Rexical::VERSION) do |p|
10
- p.readme_file = 'README.rdoc'
11
- p.history_file = 'CHANGELOG.rdoc'
12
- p.developer('Aaron Patterson', 'aaronp@rubyforge.org')
13
- p.rubyforge_name = 'ruby-rex'
14
- p.extra_rdoc_files = FileList['*.rdoc']
6
+ Hoe.spec 'rexical' do
7
+ self.readme_file = 'README.rdoc'
8
+ self.history_file = 'CHANGELOG.rdoc'
9
+ developer('Aaron Patterson', 'aaronp@rubyforge.org')
10
+ self.rubyforge_name = 'ruby-rex'
11
+ self.extra_rdoc_files = FileList['*.rdoc']
15
12
  end
16
13
 
17
14
  namespace :gem do
@@ -1,3 +1,8 @@
1
1
  require 'rexical/generator'
2
- require 'rexical/info'
3
2
  require 'rexical/rexcmd'
3
+
4
+ module Rexical
5
+ VERSION = "1.0.4"
6
+ Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
7
+ Mailto = 'arima.yasuhiro@nifty.com'
8
+ end
@@ -95,6 +95,7 @@ module Rexical
95
95
  end
96
96
  end
97
97
  expr = st[0,ndx]
98
+ expr.gsub!('\ ', ' ')
98
99
  key = '{' + key + '}'
99
100
  @macro.each_pair do |k, e|
100
101
  expr.gsub!(k) { |m| e }
@@ -314,47 +315,39 @@ REX_UTIL = <<-REX_EOT
314
315
 
315
316
  class ScanError < StandardError ; end
316
317
 
317
- attr_reader :lineno
318
- attr_reader :filename
318
+ attr_reader :lineno
319
+ attr_reader :filename
320
+ attr_accessor :state
319
321
 
320
- def scan_setup ; end
322
+ def scan_setup(str)
323
+ @ss = StringScanner.new(str)
324
+ @lineno = 1
325
+ @state = nil
326
+ end
321
327
 
322
- def action &block
328
+ def action(&block)
323
329
  yield
324
330
  end
325
331
 
326
- def scan_str( str )
327
- scan_evaluate str
332
+ def scan_str(str)
333
+ scan_setup(str)
328
334
  do_parse
329
335
  end
330
336
 
331
337
  def load_file( filename )
332
338
  @filename = filename
333
339
  open(filename, "r") do |f|
334
- scan_evaluate f.read
340
+ scan_setup(f.read)
335
341
  end
336
342
  end
337
343
 
338
344
  def scan_file( filename )
339
- load_file filename
345
+ load_file(filename)
340
346
  do_parse
341
347
  end
342
348
 
343
349
  REX_EOT
344
350
 
345
- REX_NEXTTOKEN_DEBUG = <<-REX_EOT
346
- def next_token
347
- p token = @rex_tokens.shift
348
- token
349
- end
350
- REX_EOT
351
-
352
- REX_NEXTTOKEN = <<-REX_EOT
353
- def next_token
354
- @rex_tokens.shift
355
- end
356
- REX_EOT
357
-
358
351
  REX_STUB = <<-REX_EOT
359
352
 
360
353
  if __FILE__ == $0
@@ -409,83 +402,80 @@ REX_EOT
409
402
  ## utility method
410
403
  f.print REX_UTIL
411
404
 
412
- if @opt['--debug']
413
- f.print REX_NEXTTOKEN_DEBUG
414
- else
415
- f.print REX_NEXTTOKEN
416
- end
417
-
418
405
  ## scanner method
419
406
 
420
407
  f.print <<-REX_EOT
421
408
 
422
- def scan_evaluate( str )
423
- scan_setup
424
- @rex_tokens = []
425
- @lineno = 1
426
- ss = StringScanner.new(str)
427
- state = nil
428
- until ss.eos?
429
- text = ss.peek(1)
430
- @lineno += 1 if text == "\\n"
431
- case state
409
+ def next_token
410
+ return if @ss.eos?
411
+
412
+ text = @ss.peek(1)
413
+ @lineno += 1 if text == "\\n"
414
+ token = case @state
415
+ REX_EOT
416
+
417
+ exclusive_states.each do |es|
418
+ f.printf <<-REX_EOT
419
+ when #{es ? es.to_s : "nil"}
420
+ case
432
421
  REX_EOT
422
+ rules.each do |rule|
423
+ exclusive_state, start_state, rule_expr, rule_action = *rule
424
+ if es == exclusive_state
433
425
 
434
- exclusive_states.each do |es|
435
- f.printf <<-REX_EOT
436
- when #{es ? es.to_s : "nil"}
437
- case
438
- REX_EOT
439
- rules.each do |rule|
440
- exclusive_state, start_state, rule_expr, rule_action = *rule
441
- if es == exclusive_state
442
-
443
- if rule_action
444
- if start_state
445
- f.print <<-REX_EOT
446
- when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
447
- @rex_tokens.push action #{rule_action}
448
-
449
- REX_EOT
450
- else
451
- f.print <<-REX_EOT
452
- when (text = ss.scan(/#{rule_expr}/#{flag}))
453
- @rex_tokens.push action #{rule_action}
426
+ if rule_action
427
+ if start_state
428
+ f.print <<-REX_EOT
429
+ when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
430
+ action #{rule_action}
454
431
 
455
- REX_EOT
456
- end
432
+ REX_EOT
457
433
  else
458
- if start_state
459
- f.print <<-REX_EOT
460
- when (state == #{start_state}) and (text = ss.scan(/#{rule_expr}/#{flag}))
461
- ;
434
+ f.print <<-REX_EOT
435
+ when (text = @ss.scan(/#{rule_expr}/#{flag}))
436
+ action #{rule_action}
462
437
 
463
- REX_EOT
464
- else
465
- f.print <<-REX_EOT
466
- when (text = ss.scan(/#{rule_expr}/#{flag}))
467
- ;
468
-
469
- REX_EOT
470
- end
438
+ REX_EOT
471
439
  end
440
+ else
441
+ if start_state
442
+ f.print <<-REX_EOT
443
+ when (state == #{start_state}) and (text = @ss.scan(/#{rule_expr}/#{flag}))
444
+ ;
472
445
 
446
+ REX_EOT
447
+ else
448
+ f.print <<-REX_EOT
449
+ when (text = @ss.scan(/#{rule_expr}/#{flag}))
450
+ ;
451
+
452
+ REX_EOT
453
+ end
473
454
  end
474
- end
475
- f.print <<-REX_EOT
476
- else
477
- text = ss.string[ss.pos .. -1]
478
- raise ScanError, "can not match: '" + text + "'"
479
- end # if
480
455
 
481
- REX_EOT
456
+ end
482
457
  end
483
458
  f.print <<-REX_EOT
484
459
  else
485
- raise ScanError, "undefined state: '" + state.to_s + "'"
486
- end # case state
487
- end # until ss
488
- end # def scan_evaluate
460
+ text = @ss.string[@ss.pos .. -1]
461
+ raise ScanError, "can not match: '" + text + "'"
462
+ end # if
463
+
464
+ REX_EOT
465
+ end
466
+ f.print <<-REX_EOT
467
+ else
468
+ raise ScanError, "undefined state: '" + state.to_s + "'"
469
+ end # case state
470
+ REX_EOT
471
+ if @opt['--debug']
472
+ f.print <<-REX_EOT
473
+ p token
474
+ REX_EOT
475
+ end
476
+ f.print <<-REX_EOT
477
+ token
478
+ end # def next_token
489
479
 
490
480
  REX_EOT
491
481
 
@@ -34,10 +34,7 @@ class TestGenerator < Test::Unit::TestCase
34
34
  end
35
35
 
36
36
  def test_scanner_inherits
37
- rex = Rexical::Generator.new(
38
- "--independent" => true
39
- )
40
- rex.grammar_lines = StringScanner.new %q{
37
+ source = parse_lexer %q{
41
38
  class Calculator < Bar
42
39
  rule
43
40
  \d+ { [:NUMBER, text.to_i] }
@@ -45,18 +42,11 @@ rule
45
42
  end
46
43
  }
47
44
 
48
- rex.parse
49
-
50
- output = StringIO.new
51
- rex.write_scanner output
52
- assert_match 'Calculator < Bar', output.string
45
+ assert_match 'Calculator < Bar', source
53
46
  end
54
47
 
55
48
  def test_scanner_inherits_many_levels
56
- rex = Rexical::Generator.new(
57
- "--independent" => true
58
- )
59
- rex.grammar_lines = StringScanner.new %q{
49
+ source = parse_lexer %q{
60
50
  class Calculator < Foo::Bar
61
51
  rule
62
52
  \d+ { [:NUMBER, text.to_i] }
@@ -64,18 +54,11 @@ rule
64
54
  end
65
55
  }
66
56
 
67
- rex.parse
68
-
69
- output = StringIO.new
70
- rex.write_scanner output
71
- assert_match 'Calculator < Foo::Bar', output.string
57
+ assert_match 'Calculator < Foo::Bar', source
72
58
  end
73
59
 
74
60
  def test_simple_scanner
75
- rex = Rexical::Generator.new(
76
- "--independent" => true
77
- )
78
- rex.grammar_lines = StringScanner.new %q{
61
+ m = build_lexer %q{
79
62
  class Calculator
80
63
  rule
81
64
  \d+ { [:NUMBER, text.to_i] }
@@ -83,15 +66,8 @@ rule
83
66
  end
84
67
  }
85
68
 
86
- rex.parse
87
-
88
- output = StringIO.new
89
- rex.write_scanner output
90
-
91
- m = Module.new
92
- m.module_eval output.string
93
69
  calc = m::Calculator.new
94
- calc.scan_evaluate('1 2 10')
70
+ calc.scan_setup('1 2 10')
95
71
 
96
72
  assert_tokens [[:NUMBER, 1],
97
73
  [:S, ' '],
@@ -100,11 +76,21 @@ end
100
76
  [:NUMBER, 10]], calc
101
77
  end
102
78
 
79
+ def test_parses_macros_with_escapes
80
+ source = parse_lexer %q{
81
+ class Foo
82
+ macro
83
+ w [\ \t]+
84
+ rule
85
+ {w} { [:SPACE, text] }
86
+ end
87
+ }
88
+
89
+ assert source.index('@ss.scan(/[ \t]+/))')
90
+ end
91
+
103
92
  def test_simple_scanner_with_macros
104
- rex = Rexical::Generator.new(
105
- "--independent" => true
106
- )
107
- rex.grammar_lines = StringScanner.new %q{
93
+ m = build_lexer %q{
108
94
  class Calculator
109
95
  macro
110
96
  digit \d+
@@ -114,15 +100,8 @@ rule
114
100
  end
115
101
  }
116
102
 
117
- rex.parse
118
-
119
- output = StringIO.new
120
- rex.write_scanner output
121
-
122
- m = Module.new
123
- m.module_eval output.string
124
103
  calc = m::Calculator.new
125
- calc.scan_evaluate('1 2 10')
104
+ calc.scan_setup('1 2 10')
126
105
 
127
106
  assert_tokens [[:NUMBER, 1],
128
107
  [:S, ' '],
@@ -132,10 +111,7 @@ end
132
111
  end
133
112
 
134
113
  def test_nested_macros
135
- rex = Rexical::Generator.new(
136
- "--independent" => true
137
- )
138
- rex.grammar_lines = StringScanner.new %q{
114
+ source = parse_lexer %q{
139
115
  class Calculator
140
116
  macro
141
117
  nonascii [^\0-\177]
@@ -145,18 +121,11 @@ rule
145
121
  end
146
122
  }
147
123
 
148
- rex.parse
149
-
150
- output = StringIO.new
151
- rex.write_scanner output
152
- assert_match '"[^\0-\177]*"', output.string
124
+ assert_match '"[^\0-\177]*"', source
153
125
  end
154
126
 
155
127
  def test_more_nested_macros
156
- rex = Rexical::Generator.new(
157
- "--independent" => true
158
- )
159
- rex.grammar_lines = StringScanner.new %q{
128
+ source = parse_lexer %q{
160
129
  class Calculator
161
130
  macro
162
131
  nonascii [^\0-\177]
@@ -167,18 +136,85 @@ rule
167
136
  end
168
137
  }
169
138
 
139
+ assert_match '"[^\0-\177]*"', source
140
+ end
141
+
142
+ def test_changing_state_during_lexing
143
+ lexer = build_lexer %q{
144
+ class Calculator
145
+ rule
146
+ a { self.state = :B ; [:A, text] }
147
+ :B b { self.state = nil ; [:B, text] }
148
+ end
149
+ }
150
+
151
+ calc1 = lexer::Calculator.new
152
+ calc2 = lexer::Calculator.new
153
+ calc1.scan_setup('aaaaa')
154
+ calc2.scan_setup('ababa')
155
+
156
+ # Doesn't lex all 'a's
157
+ assert_raise(lexer::Calculator::ScanError) { tokens(calc1) }
158
+
159
+ # Does lex alternating 'a's and 'b's
160
+ calc2.scan_setup('ababa')
161
+
162
+ assert_tokens [[:A, 'a'],
163
+ [:B, 'b'],
164
+ [:A, 'a'],
165
+ [:B, 'b'],
166
+ [:A, 'a']], calc2
167
+ end
168
+
169
+ def test_changing_state_is_possible_between_next_token_calls
170
+ lexer = build_lexer %q{
171
+ class Calculator
172
+ rule
173
+ a { [:A, text] }
174
+ :B b { [:B, text] }
175
+ end
176
+ }
177
+
178
+ calc = lexer::Calculator.new
179
+ calc.scan_setup('ababa')
180
+
181
+ assert_equal [:A, 'a'], calc.next_token
182
+ calc.state = :B
183
+ assert_equal [:B, 'b'], calc.next_token
184
+ calc.state = nil
185
+ assert_equal [:A, 'a'], calc.next_token
186
+ calc.state = :B
187
+ assert_equal [:B, 'b'], calc.next_token
188
+ calc.state = nil
189
+ assert_equal [:A, 'a'], calc.next_token
190
+ end
191
+
192
+ def parse_lexer(str)
193
+ rex = Rexical::Generator.new("--independent" => true)
194
+ out = StringIO.new
195
+
196
+ rex.grammar_lines = StringScanner.new(str)
170
197
  rex.parse
198
+ rex.write_scanner(out)
171
199
 
172
- output = StringIO.new
173
- rex.write_scanner output
174
- assert_match '"[^\0-\177]*"', output.string
200
+ out.string
175
201
  end
176
202
 
177
- def assert_tokens expected, scanner
203
+ def build_lexer(str)
204
+ mod = Module.new
205
+ mod.module_eval(parse_lexer(str))
206
+ mod
207
+ end
208
+
209
+ def tokens(scanner)
178
210
  tokens = []
179
211
  while token = scanner.next_token
180
212
  tokens << token
181
213
  end
182
- assert_equal expected, tokens
214
+ tokens
215
+ end
216
+
217
+ def assert_tokens(expected, scanner)
218
+ assert_equal expected, tokens(scanner)
183
219
  end
184
220
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rexical
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.3
4
+ version: 1.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - Aaron Patterson
@@ -9,7 +9,7 @@ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
11
 
12
- date: 2009-06-07 00:00:00 -07:00
12
+ date: 2009-06-30 00:00:00 -07:00
13
13
  default_executable:
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
@@ -20,7 +20,7 @@ dependencies:
20
20
  requirements:
21
21
  - - ">="
22
22
  - !ruby/object:Gem::Version
23
- version: 1.12.1
23
+ version: 2.3.1
24
24
  version:
25
25
  description: |-
26
26
  Rexical is a lexical scanner generator.
@@ -49,7 +49,6 @@ files:
49
49
  - bin/rex
50
50
  - lib/rexical.rb
51
51
  - lib/rexical/generator.rb
52
- - lib/rexical/info.rb
53
52
  - lib/rexical/rexcmd.rb
54
53
  - sample/a.cmd
55
54
  - sample/b.cmd
@@ -101,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
101
100
  requirements: []
102
101
 
103
102
  rubyforge_project: ruby-rex
104
- rubygems_version: 1.3.3
103
+ rubygems_version: 1.3.4
105
104
  signing_key:
106
105
  specification_version: 3
107
106
  summary: Rexical is a lexical scanner generator
@@ -1,16 +0,0 @@
1
- #
2
- # info.rb
3
- #
4
- # Copyright (c) 2005-2006 ARIMA Yasuhiro <arima.yasuhiro@nifty.com>
5
- #
6
- # This program is free software.
7
- # You can distribute/modify this program under the terms of
8
- # the GNU Lesser General Public License version 2 or later.
9
- #
10
-
11
- module Rexical
12
- VERSION = '1.0.3'
13
- Copyright = 'Copyright (c) 2005-2006 ARIMA Yasuhiro'
14
- Mailto = 'arima.yasuhiro@nifty.com'
15
- end
16
-