oedipus_lex 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 16d5bf4e2e41af8957fcfee63012c44ee33f65df
4
- data.tar.gz: 55ec9b8e51834e8bee5f1ae27d6d4a1276c53094
3
+ metadata.gz: d0039bfa4502e60d46539650a3671e8efb4e6131
4
+ data.tar.gz: 5ce6636d84be656a55937f9c14dfc09ab1435082
5
5
  SHA512:
6
- metadata.gz: d78c2765d033065c1be87a0a18860bec9d0f7ce359218ad88db34cd5381e70ff34435e489cdcb6fa2a6091b115f16524333dda547fcb712fac36c1192d8b8553
7
- data.tar.gz: 62cc88bdb0ee37be1d38aaca8a7ad7979839d6a9602d1797f25361873a40994d394c8d44e0d0fe15d255916cb5e716d829a410161783a8e4283b90a16bb8153b
6
+ metadata.gz: 62803c0e04df04609964b62541026fd385dcfd8b8a77093b3abdd85c8f227f7bf49629709d62010a5e42c06bdbd50add8133004c7a548af073b0a03fa10cdb87
7
+ data.tar.gz: 01bcb903f4722cd8e6e77d3a119ce52982083fdcec796338afe6a7ca93b807b5ff6829c2884e49bd104ed397e9e5244cfb75911abcaff026d342ddbd50a7705c
checksums.yaml.gz.sig CHANGED
@@ -1,2 +1,2 @@
1
- ���.�n7hZ.�A�CK��/����^�<H�-,m���HU!�����1>��Zc���uֈ�P��d/��a��!m �dO�~Z� H�d%�G��2�&.���-�w��M�"j �
2
- ˢ��cq��پ�>���"���O�A�W�]p��t���{e��pq����IqEK�6���S#o/6:C*A�/�ћ�ʈ�����<�$Oqן��|Uu�^���E{���[<
1
+ kQ_]~�j���$�
2
+ �eX��zY�3�_������A:��PO���z)\163A=�+����k}ON'=��&��S����t;3]��������n� �uON1�aT-w�I�a>�>�.W7�{��|R�01��JD���Pl�~ʤ�w�ƾa�:�G 87��ޔ���|��o�;��6��s���D��΁��x����R\Z����������2|;����|'IyDŽ��Դ�"h��2m��
data.tar.gz.sig CHANGED
Binary file
data/History.rdoc CHANGED
@@ -1,3 +1,11 @@
1
+ === 2.2.0 / 2014-03-14
2
+
3
+ * 3 minor enhancements:
4
+
5
+ * Added debug rake task.
6
+ * Added rule grouping. Naive benchmarking seems to show ~15% improvement in ruby_parser.
7
+ * Refactored rule handling part of template to its own variable.
8
+
1
9
  === 2.1.0 / 2014-01-22
2
10
 
3
11
  * 3 minor enhancements:
data/README.rdoc CHANGED
@@ -59,8 +59,9 @@ resource for CS learning. Books... books are good. I like books.
59
59
  name = /\w+/
60
60
  regexp = /(\/(?:\\.|[^\/])+\/[io]?)/
61
61
 
62
- rule_section = /rules?/ NL (rule)*
62
+ rule_section = /rules?/ NL (rule|group)*
63
63
  rule = (state)? regexp (action)?
64
+ group = /:/ regexp NL (rule)+
64
65
  state = label
65
66
  | predicate
66
67
  label = /:\w+/
data/Rakefile CHANGED
@@ -69,4 +69,13 @@ task :clean do
69
69
  rm Dir["sample/*.rb"]
70
70
  end
71
71
 
72
+ task :debug do
73
+ require "oedipus_lex"
74
+ f = ENV["F"]
75
+ rex = OedipusLex.new $rex_option
76
+ rex.parse_file f
77
+
78
+ puts rex.generate
79
+ end
80
+
72
81
  # vim: syntax=ruby
data/lib/oedipus_lex.rb CHANGED
@@ -4,7 +4,7 @@ require "erb"
4
4
  require "oedipus_lex.rex"
5
5
 
6
6
  class OedipusLex
7
- VERSION = "2.1.0"
7
+ VERSION = "2.2.0"
8
8
 
9
9
  attr_accessor :class_name
10
10
  attr_accessor :header
@@ -14,6 +14,7 @@ class OedipusLex
14
14
  attr_accessor :option
15
15
  attr_accessor :rules
16
16
  attr_accessor :starts
17
+ attr_accessor :group
17
18
 
18
19
  DEFAULTS = {
19
20
  :debug => false,
@@ -32,6 +33,7 @@ class OedipusLex
32
33
  self.macros = []
33
34
  self.rules = []
34
35
  self.starts = []
36
+ self.group = nil
35
37
  end
36
38
 
37
39
  def lex_class prefix, name
@@ -67,30 +69,79 @@ class OedipusLex
67
69
  rules << [start_state, regexp, action]
68
70
  end
69
71
 
70
- def lex_rule2(*vals)
71
- raise vals.inspect
72
+ def lex_grouphead(*vals)
73
+ end_group if group
74
+ self.state = :group
75
+ self.group = vals
76
+ end
77
+
78
+ def lex_group start_state, regexp, action = nil
79
+ self.group << [start_state, regexp, action]
80
+ end
81
+
82
+ def end_group
83
+ rules << group
84
+ self.group = nil
85
+ self.state = :rule
86
+ end
87
+
88
+ def lex_groupend start_state, regexp, action = nil
89
+ end_group
90
+ lex_rule start_state, regexp, action
72
91
  end
73
92
 
74
93
  def lex_state new_state
94
+ end_group if group
75
95
  # do nothing -- lexer switches state for us
76
96
  end
77
97
 
78
98
  def generate
79
- states = rules.map(&:first).compact.uniq
80
- exclusives, inclusives = states.partition { |s| s =~ /^:[A-Z]/ }
99
+ filter = lambda { |r| Array === r.last ? nil : r.first }
100
+ _mystates = rules.map(&filter).flatten.compact.uniq
101
+ exclusives, inclusives = _mystates.partition { |s| s =~ /^:[A-Z]/ }
81
102
 
82
103
  # NOTE: doubling up assignment to remove unused var warnings in
83
104
  # ERB binding.
84
105
 
85
106
  all_states =
86
- all_states = [[nil, # non-state # eg [[nil,
87
- *inclusives], # incls # :a, :b],
107
+ all_states = [[nil, *inclusives], # nil+incls # eg [[nil, :a],
88
108
  *exclusives.map { |s| [s] }] # [excls] # [:A], [:B]]
89
109
 
90
110
  ERB.new(TEMPLATE, nil, "%").result binding
91
111
  end
92
112
 
93
- TEMPLATE = <<-'REX'.gsub(/^ {6}/, '\1')
113
+ rule = <<-'END_RULE'.chomp
114
+ % start_state, rule_expr, rule_action = *rule
115
+ % if start_state == state or (state.nil? and predicates.include? start_state) then
116
+ % if start_state and not exclusive then
117
+ % if start_state =~ /^:/ then
118
+ when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
119
+ % else
120
+ when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
121
+ % end
122
+ % else
123
+ when text = ss.scan(<%= rule_expr %>) then
124
+ % end
125
+ % if rule_action then
126
+ % case rule_action
127
+ % when /^\{/ then
128
+ action <%= rule_action %>
129
+ % when /^:/, "nil" then
130
+ [:state, <%= rule_action %>]
131
+ % else
132
+ <%= rule_action %> text
133
+ % end
134
+ % else
135
+ # do nothing
136
+ % end
137
+ % end # start_state == state
138
+ END_RULE
139
+
140
+ subrule = rule.gsub(/^ /, " ").
141
+ sub(/\*rule/, "*subrule"). # use subrules, not top level rules
142
+ sub(/start_state == state or/, "true or") # hack to prevent state checks
143
+
144
+ TEMPLATE = <<-'REX'.sub(/SUBRULE/, subrule).sub(/RULE/, rule).gsub(/^ {6}/, '\1')
94
145
  #--
95
146
  # This file is automatically generated. Do not modify it.
96
147
  # Generated by: oedipus_lex version <%= VERSION %>.
@@ -178,36 +229,22 @@ class OedipusLex
178
229
  case state
179
230
  % all_states.each do |the_states|
180
231
  % exclusive = the_states.first != nil
181
- % all_states, predicates = the_states.partition { |s| s.nil? or s.start_with? ":" }
182
- % filtered_states = the_states.select { |s| s.nil? or s.start_with? ":" }
183
- when <%= all_states.map { |s| s || "nil" }.join ", " %> then
232
+ % the_states, predicates = the_states.partition { |s| s.nil? or s.start_with? ":" }
233
+ when <%= the_states.map { |s| s || "nil" }.join ", " %> then
184
234
  case
185
- % all_states.each do |state|
235
+ % the_states.each do |state|
186
236
  % rules.each do |rule|
187
- % start_state, rule_expr, rule_action = *rule
188
- % if start_state == state or (state.nil? and predicates.include? start_state) then
189
- % if start_state and not exclusive then
190
- % if start_state =~ /^:/ then
191
- when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
192
- % else
193
- when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
194
- % end
195
- % else
196
- when text = ss.scan(<%= rule_expr %>) then
197
- % end
198
- % if rule_action then
199
- % case rule_action
200
- % when /^\{/ then
201
- action <%= rule_action %>
202
- % when /^:/, "nil" then
203
- [:state, <%= rule_action %>]
204
- % else
205
- <%= rule_action %> text
206
- % end
207
- % else
208
- # do nothing
237
+ % if Array === rule.last then
238
+ % group_re, *subrules = rule
239
+ when ss.check(<%= group_re %>) then
240
+ case
241
+ % subrules.each do |subrule|
242
+ SUBRULE
209
243
  % end
210
- % end # start_state == state
244
+ end # group <%= group_re %>
245
+ % else # rule or group
246
+ RULE
247
+ % end # rule or group
211
248
  % end # rules.each
212
249
  % end # the_states.each
213
250
  else
data/lib/oedipus_lex.rex CHANGED
@@ -1,16 +1,3 @@
1
- # [Header Part]
2
- # "class" Foo
3
- # ["option"
4
- # [options] ]
5
- # ["inner"
6
- # [methods] ]
7
- # ["macro"
8
- # [macro-name /pattern/[flags]] ]
9
- # "rule"
10
- # [:state | method_name] /pattern/[flags] [{ code } | method_name | :state]
11
- # "end"
12
- # [Footer Part]
13
-
14
1
  class OedipusLex
15
2
 
16
3
  option
@@ -51,7 +38,11 @@ rule
51
38
 
52
39
  :macro /\s+(\w+)\s+#{RE}/o { [:macro, *matches] }
53
40
 
54
- :rule /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:rule, *matches] }
41
+ :rule /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:rule, *matches] }
42
+ :rule /\s*:[\ \t]*#{RE}/o { [:grouphead, *matches] }
43
+ :group /\s*:[\ \t]*#{RE}/o { [:grouphead, *matches] }
44
+ :group /\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:group, *matches] }
45
+ :group /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:groupend, *matches] }
55
46
 
56
47
  :END /\n+/ # do nothing
57
48
  :END /.*/ { [:end, text] }
@@ -1,22 +1,9 @@
1
1
  #--
2
2
  # This file is automatically generated. Do not modify it.
3
- # Generated by: oedipus_lex version 2.0.0.
3
+ # Generated by: oedipus_lex version 2.1.0.
4
4
  # Source: lib/oedipus_lex.rex
5
5
  #++
6
6
 
7
- # [Header Part]
8
- # "class" Foo
9
- # ["option"
10
- # [options] ]
11
- # ["inner"
12
- # [methods] ]
13
- # ["macro"
14
- # [macro-name /pattern/[flags]] ]
15
- # "rule"
16
- # [:state | method_name] /pattern/[flags] [{ code } | method_name | :state]
17
- # "end"
18
- # [Footer Part]
19
-
20
7
  class OedipusLex
21
8
  require 'strscan'
22
9
 
@@ -78,7 +65,7 @@ class OedipusLex
78
65
  until ss.eos? or token do
79
66
  token =
80
67
  case state
81
- when nil, :option, :inner, :start, :macro, :rule then
68
+ when nil, :option, :inner, :start, :macro, :rule, :group then
82
69
  case
83
70
  when text = ss.scan(/options?.*/) then
84
71
  [:state, :option]
@@ -116,6 +103,14 @@ class OedipusLex
116
103
  action { [:macro, *matches] }
117
104
  when (state == :rule) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
118
105
  action { [:rule, *matches] }
106
+ when (state == :rule) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
107
+ action { [:grouphead, *matches] }
108
+ when (state == :group) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
109
+ action { [:grouphead, *matches] }
110
+ when (state == :group) && (text = ss.scan(/\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
111
+ action { [:group, *matches] }
112
+ when (state == :group) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
113
+ action { [:groupend, *matches] }
119
114
  else
120
115
  text = ss.string[ss.pos .. -1]
121
116
  raise ScanError, "can not match (#{state.inspect}): '#{text}'"
@@ -420,6 +420,54 @@ class TestOedipusLex < Minitest::Test
420
420
  assert_match "when :ARG then", ruby
421
421
  end
422
422
 
423
+ def test_simple_scanner_group
424
+ src = <<-'REX'
425
+ class Calculator
426
+ rules
427
+
428
+ : /\d/
429
+ | /\d+\.\d+/ { [:float, text.to_f] }
430
+ | /\d+/ { [:int, text.to_i] }
431
+ /\s+/
432
+ end
433
+ REX
434
+
435
+ ruby = generate_lexer src
436
+
437
+ assert_match "when ss.check(/\\d/) then", ruby
438
+ assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
439
+ assert_match "when text = ss.scan(/\\d+/) then", ruby
440
+ assert_match "end # group /\\d/", ruby
441
+ end
442
+
443
+ def test_simple_scanner_group_I_am_dumb
444
+ src = <<-'REX'
445
+ class Calculator
446
+ rules
447
+
448
+ : /\d/
449
+ | /\d+\.\d+/ { [:float, text.to_f] }
450
+ | /\d+/ { [:int, text.to_i] }
451
+ : /\+/
452
+ | xx? /\+whatever/ { [:x, text] }
453
+ | :x /\+\d+/ { [:y, text] }
454
+ /\s+/
455
+ end
456
+ REX
457
+
458
+ ruby = generate_lexer src
459
+
460
+ assert_match "when ss.check(/\\d/) then", ruby
461
+ assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
462
+ assert_match "when text = ss.scan(/\\d+/) then", ruby
463
+ assert_match "end # group /\\d/", ruby
464
+
465
+ assert_match "when ss.check(/\\+/) then", ruby
466
+ assert_match "when xx? && (text = ss.scan(/\\+whatever/)) then", ruby
467
+ assert_match "when (state == :x) && (text = ss.scan(/\\+\\d+/)) then", ruby
468
+ assert_match "end # group /\\d/", ruby
469
+ end
470
+
423
471
  def test_generator_start
424
472
  src = <<-'REX'
425
473
  class Calculator
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: oedipus_lex
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.0
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Ryan Davis
@@ -29,7 +29,7 @@ cert_chain:
29
29
  Y4evBVezr3SjXz08vPqRO5YRdO3zfeMT8gBjRqZjWJGMZ2lD4XNfrs7eky74CyZw
30
30
  xx3n58i0lQkBE1EpKE0lFu/y
31
31
  -----END CERTIFICATE-----
32
- date: 2014-01-22 00:00:00.000000000 Z
32
+ date: 2014-03-14 00:00:00.000000000 Z
33
33
  dependencies:
34
34
  - !ruby/object:Gem::Dependency
35
35
  name: minitest
@@ -37,14 +37,14 @@ dependencies:
37
37
  requirements:
38
38
  - - ~>
39
39
  - !ruby/object:Gem::Version
40
- version: '5.2'
40
+ version: '5.3'
41
41
  type: :development
42
42
  prerelease: false
43
43
  version_requirements: !ruby/object:Gem::Requirement
44
44
  requirements:
45
45
  - - ~>
46
46
  - !ruby/object:Gem::Version
47
- version: '5.2'
47
+ version: '5.3'
48
48
  - !ruby/object:Gem::Dependency
49
49
  name: rdoc
50
50
  requirement: !ruby/object:Gem::Requirement
@@ -65,14 +65,14 @@ dependencies:
65
65
  requirements:
66
66
  - - ~>
67
67
  - !ruby/object:Gem::Version
68
- version: '3.8'
68
+ version: '3.10'
69
69
  type: :development
70
70
  prerelease: false
71
71
  version_requirements: !ruby/object:Gem::Requirement
72
72
  requirements:
73
73
  - - ~>
74
74
  - !ruby/object:Gem::Version
75
- version: '3.8'
75
+ version: '3.10'
76
76
  description: |-
77
77
  Oedipus Lex is a lexer generator in the same family as Rexical and
78
78
  Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was
@@ -107,7 +107,6 @@ extra_rdoc_files:
107
107
  - sample/error1.txt
108
108
  files:
109
109
  - .autotest
110
- - .gemtest
111
110
  - History.rdoc
112
111
  - Manifest.txt
113
112
  - README.rdoc
@@ -134,6 +133,7 @@ files:
134
133
  - sample/xhtmlparser.rex
135
134
  - sample/xhtmlparser.xhtml
136
135
  - test/test_oedipus_lex.rb
136
+ - .gemtest
137
137
  homepage: http://github.com/seattlerb/oedipus_lex
138
138
  licenses:
139
139
  - MIT
@@ -155,8 +155,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
155
155
  - !ruby/object:Gem::Version
156
156
  version: '0'
157
157
  requirements: []
158
- rubyforge_project: oedipus_lex
159
- rubygems_version: 2.2.1
158
+ rubyforge_project:
159
+ rubygems_version: 2.0.14
160
160
  signing_key:
161
161
  specification_version: 4
162
162
  summary: Oedipus Lex is a lexer generator in the same family as Rexical and Rex
metadata.gz.sig CHANGED
Binary file