rexical 1.0.5 → 1.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -22,6 +22,7 @@ o -s --stub - append stub code for debug
22
22
  o -i --ignorecase - ignore char case
23
23
  o -C --check-only - syntax check only
24
24
  o - --independent - independent mode
25
+ o - --matcheos - allow match against end of string
25
26
  o -d --debug - print debug information
26
27
  o -h --help - print this message and quit
27
28
  o - --version - print version and quit
@@ -65,7 +66,8 @@ EOT
65
66
  @cmd = File.basename($0, ".rb")
66
67
  tmp = OPTIONS.lines.collect do |line|
67
68
  next if /\A\s*\z/ === line
68
- disp, sopt, lopt, takearg, doc = line.strip.split(/\s+/, 5)
69
+ # disp, sopt, lopt, takearg, doc
70
+ _, sopt, lopt, takearg, _ = line.strip.split(/\s+/, 5)
69
71
  a = []
70
72
  a.push lopt unless lopt == '-'
71
73
  a.push sopt unless sopt == '-'
@@ -1,7 +1,7 @@
1
1
  #
2
2
  # DO NOT MODIFY!!!!
3
3
  # This file is automatically generated by racc 1.4.4
4
- # from racc grammer file "calc3.racc".
4
+ # from racc grammar file "calc3.racc".
5
5
  #
6
6
 
7
7
  require 'racc/parser'
@@ -7,7 +7,7 @@ class Error1
7
7
  macro
8
8
  BLANK [\ \t]+
9
9
  rule
10
- BLANK # no action
10
+ {BLANK} # no action
11
11
  \d+ { [:digit, text.to_i] }
12
12
  \w+ { [:word, text] }
13
13
  \n
@@ -7,7 +7,7 @@ class Error2
7
7
  macro
8
8
  BLANK [\ \t]+
9
9
  rule
10
- BLANK # no action
10
+ {BLANK} # no action
11
11
  \d+ { [:digit, text.to_i] }
12
12
  \w+ { [:word, text] }
13
13
  \n
@@ -7,7 +7,7 @@ class Sample
7
7
  macro
8
8
  BLANK [\ \t]+
9
9
  rule
10
- BLANK # no action
10
+ {BLANK} # no action
11
11
  \d+ { [:digit, text.to_i] }
12
12
  \w+ { [:word, text] }
13
13
  \n
@@ -1,100 +1,92 @@
1
- #
2
- # DO NOT MODIFY!!!!
3
- # This file is automatically generated by rex 1.0.0
4
- # from lexical definition file "sample.rex".
5
- #
6
-
7
- require 'racc/parser'
8
- #
9
- # sample.rex
10
- # lexical definition sample for rex
11
- #
12
-
13
- class Sample < Racc::Parser
14
- require 'strscan'
15
-
16
- class ScanError < StandardError ; end
17
-
18
- attr_reader :lineno
19
- attr_reader :filename
20
-
21
- def scan_setup ; end
22
-
23
- def action &block
24
- yield
25
- end
26
-
27
- def scan_str( str )
28
- scan_evaluate str
29
- do_parse
30
- end
31
-
32
- def load_file( filename )
33
- @filename = filename
34
- open(filename, "r") do |f|
35
- scan_evaluate f.read
36
- end
37
- end
38
-
39
- def scan_file( filename )
40
- load_file filename
41
- do_parse
42
- end
43
-
44
- def next_token
45
- @rex_tokens.shift
46
- end
47
-
48
- def scan_evaluate( str )
49
- scan_setup
50
- @rex_tokens = []
51
- @lineno = 1
52
- ss = StringScanner.new(str)
53
- state = nil
54
- until ss.eos?
55
- text = ss.peek(1)
56
- @lineno += 1 if text == "\n"
57
- case state
58
- when nil
59
- case
60
- when (text = ss.scan(/BLANK/))
61
- ;
62
-
63
- when (text = ss.scan(/\d+/))
64
- @rex_tokens.push action { [:digit, text.to_i] }
65
-
66
- when (text = ss.scan(/\w+/))
67
- @rex_tokens.push action { [:word, text] }
68
-
69
- when (text = ss.scan(/\n/))
70
- ;
71
-
72
- when (text = ss.scan(/./))
73
- @rex_tokens.push action { [text, text] }
74
-
75
- else
76
- text = ss.string[ss.pos .. -1]
77
- raise ScanError, "can not match: '" + text + "'"
78
- end # if
79
-
80
- else
81
- raise ScanError, "undefined state: '" + state.to_s + "'"
82
- end # case state
83
- end # until ss
84
- end # def scan_evaluate
85
-
86
- end # class
87
-
88
- if __FILE__ == $0
89
- exit if ARGV.size != 1
90
- filename = ARGV.shift
91
- rex = Sample.new
92
- begin
93
- rex.load_file filename
94
- while token = rex.next_token
95
- p token
96
- end
97
- rescue
98
- $stderr.printf "%s:%d:%s\n", rex.filename, rex.lineno, $!.message
99
- end
100
- end
1
+ #--
2
+ # DO NOT MODIFY!!!!
3
+ # This file is automatically generated by rex 1.0.5
4
+ # from lexical definition file "sample/sample.rex".
5
+ #++
6
+
7
+ require 'racc/parser'
8
+ #
9
+ # sample.rex
10
+ # lexical definition sample for rex
11
+ #
12
+
13
+ class Sample < Racc::Parser
14
+ require 'strscan'
15
+
16
+ class ScanError < StandardError ; end
17
+
18
+ attr_reader :lineno
19
+ attr_reader :filename
20
+ attr_accessor :state
21
+
22
+ def scan_setup(str)
23
+ @ss = StringScanner.new(str)
24
+ @lineno = 1
25
+ @state = nil
26
+ end
27
+
28
+ def action
29
+ yield
30
+ end
31
+
32
+ def scan_str(str)
33
+ scan_setup(str)
34
+ do_parse
35
+ end
36
+ alias :scan :scan_str
37
+
38
+ def load_file( filename )
39
+ @filename = filename
40
+ open(filename, "r") do |f|
41
+ scan_setup(f.read)
42
+ end
43
+ end
44
+
45
+ def scan_file( filename )
46
+ load_file(filename)
47
+ do_parse
48
+ end
49
+
50
+
51
+ def next_token
52
+ return if @ss.eos?
53
+
54
+ # skips empty actions
55
+ until token = _next_token or @ss.eos?; end
56
+ token
57
+ end
58
+
59
+ def _next_token
60
+ text = @ss.peek(1)
61
+ @lineno += 1 if text == "\n"
62
+ token = case @state
63
+ when nil
64
+ case
65
+ when (text = @ss.scan(/[ \t]+/))
66
+ ;
67
+
68
+ when (text = @ss.scan(/\d+/))
69
+ action { [:digit, text.to_i] }
70
+
71
+ when (text = @ss.scan(/\w+/))
72
+ action { [:word, text] }
73
+
74
+ when (text = @ss.scan(/\n/))
75
+ ;
76
+
77
+ when (text = @ss.scan(/./))
78
+ action { [text, text] }
79
+
80
+
81
+ else
82
+ text = @ss.string[@ss.pos .. -1]
83
+ raise ScanError, "can not match: '" + text + "'"
84
+ end # if
85
+
86
+ else
87
+ raise ScanError, "undefined state: '" + state.to_s + "'"
88
+ end # case state
89
+ token
90
+ end # def _next_token
91
+
92
+ end # class
@@ -5,8 +5,12 @@ class C < SomethingElse
5
5
  macro
6
6
  w [\s\r\n\f]*
7
7
 
8
+ rule
9
+
8
10
  # [:state] pattern [actions]
9
11
 
10
12
  {w}~={w} { [:INCLUDES, text] }
11
13
 
12
14
  end
15
+ end
16
+ end
@@ -1,9 +1,11 @@
1
- require 'test/unit'
1
+ gem "minitest"
2
+ require 'minitest/autorun'
2
3
  require 'tempfile'
3
4
  require 'rexical'
4
5
  require 'stringio'
6
+ require 'open3'
5
7
 
6
- class TestGenerator < Test::Unit::TestCase
8
+ class TestGenerator < Minitest::Test
7
9
  def test_header_is_written_after_module
8
10
  rex = Rexical::Generator.new(
9
11
  "--independent" => true
@@ -25,6 +27,29 @@ class TestGenerator < Test::Unit::TestCase
25
27
  assert_equal '#++', comments.last
26
28
  end
27
29
 
30
+ def test_rubocop_security
31
+ rex = Rexical::Generator.new(
32
+ "--independent" => true
33
+ )
34
+ rex.grammar_file = File.join File.dirname(__FILE__), 'assets', 'test.rex'
35
+ rex.read_grammar
36
+ rex.parse
37
+
38
+ output = Tempfile.new(["rex_output", ".rb"])
39
+ begin
40
+ rex.write_scanner output
41
+ output.close
42
+
43
+ stdin, stdoe, wait_thr = Open3.popen2e "rubocop --only Security #{output.path}"
44
+ if ! wait_thr.value.success?
45
+ fail stdoe.read
46
+ end
47
+ ensure
48
+ output.close
49
+ output.unlink
50
+ end
51
+ end
52
+
28
53
  def test_read_non_existent_file
29
54
  rex = Rexical::Generator.new(nil)
30
55
  rex.grammar_file = 'non_existent_file'
@@ -71,6 +96,22 @@ end
71
96
  assert_match 'Calculator < Foo::Bar', source
72
97
  end
73
98
 
99
+ def test_stateful_lexer
100
+ m = build_lexer %q{
101
+ class Foo
102
+ rule
103
+ \d { @state = :digit; [:foo, text] }
104
+ :digit \w { @state = nil; [:w, text] }
105
+ end
106
+ }
107
+ scanner = m::Foo.new
108
+ scanner.scan_setup('1w1')
109
+ assert_tokens [
110
+ [:foo, '1'],
111
+ [:w, 'w'],
112
+ [:foo, '1']], scanner
113
+ end
114
+
74
115
  def test_simple_scanner
75
116
  m = build_lexer %q{
76
117
  class Calculator
@@ -185,7 +226,7 @@ end
185
226
  calc2.scan_setup('ababa')
186
227
 
187
228
  # Doesn't lex all 'a's
188
- assert_raise(lexer::Calculator::ScanError) { tokens(calc1) }
229
+ assert_raises(lexer::Calculator::ScanError) { tokens(calc1) }
189
230
 
190
231
  # Does lex alternating 'a's and 'b's
191
232
  calc2.scan_setup('ababa')
@@ -219,6 +260,21 @@ end
219
260
  calc.state = nil
220
261
  assert_equal [:A, 'a'], calc.next_token
221
262
  end
263
+ def test_match_eos
264
+ lexer = build_lexer %q{
265
+ class Calculator
266
+ option
267
+ matcheos
268
+ rule
269
+ a { [:A, text] }
270
+ $ { [:EOF, ""] }
271
+ :B b { [:B, text] }
272
+ }
273
+ calc = lexer::Calculator.new
274
+ calc.scan_setup("a")
275
+ assert_equal [:A, 'a'], calc.next_token
276
+ assert_equal [:EOF, ""], calc.next_token
277
+ end
222
278
 
223
279
  def parse_lexer(str)
224
280
  rex = Rexical::Generator.new("--independent" => true)
metadata CHANGED
@@ -1,54 +1,79 @@
1
- --- !ruby/object:Gem::Specification
1
+ --- !ruby/object:Gem::Specification
2
2
  name: rexical
3
- version: !ruby/object:Gem::Version
4
- prerelease: false
5
- segments:
6
- - 1
7
- - 0
8
- - 5
9
- version: 1.0.5
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.7
10
5
  platform: ruby
11
- authors:
6
+ authors:
12
7
  - Aaron Patterson
13
8
  autorequire:
14
9
  bindir: bin
15
10
  cert_chain: []
16
-
17
- date: 2010-12-08 00:00:00 -08:00
18
- default_executable:
19
- dependencies:
20
- - !ruby/object:Gem::Dependency
21
- name: hoe
11
+ date: 2019-08-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: rubocop
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 0.74.0
20
+ type: :development
22
21
  prerelease: false
23
- requirement: &id001 !ruby/object:Gem::Requirement
24
- none: false
25
- requirements:
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 0.74.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: rdoc
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
26
31
  - - ">="
27
- - !ruby/object:Gem::Version
28
- segments:
29
- - 2
30
- - 6
31
- - 2
32
- version: 2.6.2
32
+ - !ruby/object:Gem::Version
33
+ version: '4.0'
34
+ - - "<"
35
+ - !ruby/object:Gem::Version
36
+ version: '7'
33
37
  type: :development
34
- version_requirements: *id001
38
+ prerelease: false
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ version: '4.0'
44
+ - - "<"
45
+ - !ruby/object:Gem::Version
46
+ version: '7'
47
+ - !ruby/object:Gem::Dependency
48
+ name: hoe
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '3.18'
54
+ type: :development
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '3.18'
35
61
  description: |-
36
- Rexical is a lexical scanner generator.
37
- It is written in Ruby itself, and generates Ruby program.
38
- It is designed for use with Racc.
39
- email:
62
+ Rexical is a lexical scanner generator that is used with
63
+ Racc to generate Ruby programs.
64
+ Rexical is written in Ruby.
65
+ email:
40
66
  - aaronp@rubyforge.org
41
- executables:
67
+ executables:
42
68
  - rex
43
69
  extensions: []
44
-
45
- extra_rdoc_files:
46
- - Manifest.txt
70
+ extra_rdoc_files:
47
71
  - CHANGELOG.rdoc
48
72
  - DOCUMENTATION.en.rdoc
49
73
  - DOCUMENTATION.ja.rdoc
74
+ - Manifest.txt
50
75
  - README.rdoc
51
- files:
76
+ files:
52
77
  - CHANGELOG.rdoc
53
78
  - DOCUMENTATION.en.rdoc
54
79
  - DOCUMENTATION.ja.rdoc
@@ -85,38 +110,30 @@ files:
85
110
  - test/rex-20060125.rb
86
111
  - test/rex-20060511.rb
87
112
  - test/test_generator.rb
88
- has_rdoc: true
89
113
  homepage: http://github.com/tenderlove/rexical/tree/master
90
- licenses: []
91
-
114
+ licenses:
115
+ - MIT
116
+ metadata: {}
92
117
  post_install_message:
93
- rdoc_options:
94
- - --main
118
+ rdoc_options:
119
+ - "--main"
95
120
  - README.rdoc
96
- require_paths:
121
+ require_paths:
97
122
  - lib
98
- required_ruby_version: !ruby/object:Gem::Requirement
99
- none: false
100
- requirements:
123
+ required_ruby_version: !ruby/object:Gem::Requirement
124
+ requirements:
101
125
  - - ">="
102
- - !ruby/object:Gem::Version
103
- segments:
104
- - 0
105
- version: "0"
106
- required_rubygems_version: !ruby/object:Gem::Requirement
107
- none: false
108
- requirements:
126
+ - !ruby/object:Gem::Version
127
+ version: '0'
128
+ required_rubygems_version: !ruby/object:Gem::Requirement
129
+ requirements:
109
130
  - - ">="
110
- - !ruby/object:Gem::Version
111
- segments:
112
- - 0
113
- version: "0"
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
114
133
  requirements: []
115
-
116
- rubyforge_project: ruby-rex
117
- rubygems_version: 1.3.7
134
+ rubygems_version: 3.0.3
118
135
  signing_key:
119
- specification_version: 3
120
- summary: Rexical is a lexical scanner generator
121
- test_files:
122
- - test/test_generator.rb
136
+ specification_version: 4
137
+ summary: Rexical is a lexical scanner generator that is used with Racc to generate
138
+ Ruby programs
139
+ test_files: []