oedipus_lex 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -2
- data.tar.gz.sig +0 -0
- data/History.rdoc +8 -0
- data/README.rdoc +2 -1
- data/Rakefile +9 -0
- data/lib/oedipus_lex.rb +72 -35
- data/lib/oedipus_lex.rex +5 -14
- data/lib/oedipus_lex.rex.rb +10 -15
- data/test/test_oedipus_lex.rb +48 -0
- metadata +9 -9
- metadata.gz.sig +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0039bfa4502e60d46539650a3671e8efb4e6131
|
4
|
+
data.tar.gz: 5ce6636d84be656a55937f9c14dfc09ab1435082
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 62803c0e04df04609964b62541026fd385dcfd8b8a77093b3abdd85c8f227f7bf49629709d62010a5e42c06bdbd50add8133004c7a548af073b0a03fa10cdb87
|
7
|
+
data.tar.gz: 01bcb903f4722cd8e6e77d3a119ce52982083fdcec796338afe6a7ca93b807b5ff6829c2884e49bd104ed397e9e5244cfb75911abcaff026d342ddbd50a7705c
|
checksums.yaml.gz.sig
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
k�Q_]~�j���$�
|
2
|
+
�eX��z�Y�3�_������A:��PO���z)\163A=�+����k}�ON�'=��&��S����t;3]��������n� �uON1�aT-w�I�a>�>�.W7�{��|R�01��JD���P�l�~ʤ�w�ƾa�:�G �87��ޔ���|��o�;��6��s���D����x����R\Z����������2|;����|'�I�yDŽ��Դ�"�h��2m��
|
data.tar.gz.sig
CHANGED
Binary file
|
data/History.rdoc
CHANGED
@@ -1,3 +1,11 @@
|
|
1
|
+
=== 2.2.0 / 2014-03-14
|
2
|
+
|
3
|
+
* 3 minor enhancements:
|
4
|
+
|
5
|
+
* Added debug rake task.
|
6
|
+
* Added rule grouping. Naive benchmarking seems to show ~15% improvement in ruby_parser.
|
7
|
+
* Refactored rule handling part of template to its own variable.
|
8
|
+
|
1
9
|
=== 2.1.0 / 2014-01-22
|
2
10
|
|
3
11
|
* 3 minor enhancements:
|
data/README.rdoc
CHANGED
@@ -59,8 +59,9 @@ resource for CS learning. Books... books are good. I like books.
|
|
59
59
|
name = /\w+/
|
60
60
|
regexp = /(\/(?:\\.|[^\/])+\/[io]?)/
|
61
61
|
|
62
|
-
rule_section = /rules?/ NL (rule)*
|
62
|
+
rule_section = /rules?/ NL (rule|group)*
|
63
63
|
rule = (state)? regexp (action)?
|
64
|
+
group = /:/ regexp NL (rule)+
|
64
65
|
state = label
|
65
66
|
| predicate
|
66
67
|
label = /:\w+/
|
data/Rakefile
CHANGED
data/lib/oedipus_lex.rb
CHANGED
@@ -4,7 +4,7 @@ require "erb"
|
|
4
4
|
require "oedipus_lex.rex"
|
5
5
|
|
6
6
|
class OedipusLex
|
7
|
-
VERSION = "2.
|
7
|
+
VERSION = "2.2.0"
|
8
8
|
|
9
9
|
attr_accessor :class_name
|
10
10
|
attr_accessor :header
|
@@ -14,6 +14,7 @@ class OedipusLex
|
|
14
14
|
attr_accessor :option
|
15
15
|
attr_accessor :rules
|
16
16
|
attr_accessor :starts
|
17
|
+
attr_accessor :group
|
17
18
|
|
18
19
|
DEFAULTS = {
|
19
20
|
:debug => false,
|
@@ -32,6 +33,7 @@ class OedipusLex
|
|
32
33
|
self.macros = []
|
33
34
|
self.rules = []
|
34
35
|
self.starts = []
|
36
|
+
self.group = nil
|
35
37
|
end
|
36
38
|
|
37
39
|
def lex_class prefix, name
|
@@ -67,30 +69,79 @@ class OedipusLex
|
|
67
69
|
rules << [start_state, regexp, action]
|
68
70
|
end
|
69
71
|
|
70
|
-
def
|
71
|
-
|
72
|
+
def lex_grouphead(*vals)
|
73
|
+
end_group if group
|
74
|
+
self.state = :group
|
75
|
+
self.group = vals
|
76
|
+
end
|
77
|
+
|
78
|
+
def lex_group start_state, regexp, action = nil
|
79
|
+
self.group << [start_state, regexp, action]
|
80
|
+
end
|
81
|
+
|
82
|
+
def end_group
|
83
|
+
rules << group
|
84
|
+
self.group = nil
|
85
|
+
self.state = :rule
|
86
|
+
end
|
87
|
+
|
88
|
+
def lex_groupend start_state, regexp, action = nil
|
89
|
+
end_group
|
90
|
+
lex_rule start_state, regexp, action
|
72
91
|
end
|
73
92
|
|
74
93
|
def lex_state new_state
|
94
|
+
end_group if group
|
75
95
|
# do nothing -- lexer switches state for us
|
76
96
|
end
|
77
97
|
|
78
98
|
def generate
|
79
|
-
|
80
|
-
|
99
|
+
filter = lambda { |r| Array === r.last ? nil : r.first }
|
100
|
+
_mystates = rules.map(&filter).flatten.compact.uniq
|
101
|
+
exclusives, inclusives = _mystates.partition { |s| s =~ /^:[A-Z]/ }
|
81
102
|
|
82
103
|
# NOTE: doubling up assignment to remove unused var warnings in
|
83
104
|
# ERB binding.
|
84
105
|
|
85
106
|
all_states =
|
86
|
-
all_states = [[nil,
|
87
|
-
*inclusives], # incls # :a, :b],
|
107
|
+
all_states = [[nil, *inclusives], # nil+incls # eg [[nil, :a],
|
88
108
|
*exclusives.map { |s| [s] }] # [excls] # [:A], [:B]]
|
89
109
|
|
90
110
|
ERB.new(TEMPLATE, nil, "%").result binding
|
91
111
|
end
|
92
112
|
|
93
|
-
|
113
|
+
rule = <<-'END_RULE'.chomp
|
114
|
+
% start_state, rule_expr, rule_action = *rule
|
115
|
+
% if start_state == state or (state.nil? and predicates.include? start_state) then
|
116
|
+
% if start_state and not exclusive then
|
117
|
+
% if start_state =~ /^:/ then
|
118
|
+
when (state == <%= start_state %>) && (text = ss.scan(<%= rule_expr %>)) then
|
119
|
+
% else
|
120
|
+
when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
|
121
|
+
% end
|
122
|
+
% else
|
123
|
+
when text = ss.scan(<%= rule_expr %>) then
|
124
|
+
% end
|
125
|
+
% if rule_action then
|
126
|
+
% case rule_action
|
127
|
+
% when /^\{/ then
|
128
|
+
action <%= rule_action %>
|
129
|
+
% when /^:/, "nil" then
|
130
|
+
[:state, <%= rule_action %>]
|
131
|
+
% else
|
132
|
+
<%= rule_action %> text
|
133
|
+
% end
|
134
|
+
% else
|
135
|
+
# do nothing
|
136
|
+
% end
|
137
|
+
% end # start_state == state
|
138
|
+
END_RULE
|
139
|
+
|
140
|
+
subrule = rule.gsub(/^ /, " ").
|
141
|
+
sub(/\*rule/, "*subrule"). # use subrules, not top level rules
|
142
|
+
sub(/start_state == state or/, "true or") # hack to prevent state checks
|
143
|
+
|
144
|
+
TEMPLATE = <<-'REX'.sub(/SUBRULE/, subrule).sub(/RULE/, rule).gsub(/^ {6}/, '\1')
|
94
145
|
#--
|
95
146
|
# This file is automatically generated. Do not modify it.
|
96
147
|
# Generated by: oedipus_lex version <%= VERSION %>.
|
@@ -178,36 +229,22 @@ class OedipusLex
|
|
178
229
|
case state
|
179
230
|
% all_states.each do |the_states|
|
180
231
|
% exclusive = the_states.first != nil
|
181
|
-
%
|
182
|
-
|
183
|
-
when <%= all_states.map { |s| s || "nil" }.join ", " %> then
|
232
|
+
% the_states, predicates = the_states.partition { |s| s.nil? or s.start_with? ":" }
|
233
|
+
when <%= the_states.map { |s| s || "nil" }.join ", " %> then
|
184
234
|
case
|
185
|
-
%
|
235
|
+
% the_states.each do |state|
|
186
236
|
% rules.each do |rule|
|
187
|
-
%
|
188
|
-
%
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
when <%= start_state %> && (text = ss.scan(<%= rule_expr %>)) then
|
194
|
-
% end
|
195
|
-
% else
|
196
|
-
when text = ss.scan(<%= rule_expr %>) then
|
197
|
-
% end
|
198
|
-
% if rule_action then
|
199
|
-
% case rule_action
|
200
|
-
% when /^\{/ then
|
201
|
-
action <%= rule_action %>
|
202
|
-
% when /^:/, "nil" then
|
203
|
-
[:state, <%= rule_action %>]
|
204
|
-
% else
|
205
|
-
<%= rule_action %> text
|
206
|
-
% end
|
207
|
-
% else
|
208
|
-
# do nothing
|
237
|
+
% if Array === rule.last then
|
238
|
+
% group_re, *subrules = rule
|
239
|
+
when ss.check(<%= group_re %>) then
|
240
|
+
case
|
241
|
+
% subrules.each do |subrule|
|
242
|
+
SUBRULE
|
209
243
|
% end
|
210
|
-
|
244
|
+
end # group <%= group_re %>
|
245
|
+
% else # rule or group
|
246
|
+
RULE
|
247
|
+
% end # rule or group
|
211
248
|
% end # rules.each
|
212
249
|
% end # the_states.each
|
213
250
|
else
|
data/lib/oedipus_lex.rex
CHANGED
@@ -1,16 +1,3 @@
|
|
1
|
-
# [Header Part]
|
2
|
-
# "class" Foo
|
3
|
-
# ["option"
|
4
|
-
# [options] ]
|
5
|
-
# ["inner"
|
6
|
-
# [methods] ]
|
7
|
-
# ["macro"
|
8
|
-
# [macro-name /pattern/[flags]] ]
|
9
|
-
# "rule"
|
10
|
-
# [:state | method_name] /pattern/[flags] [{ code } | method_name | :state]
|
11
|
-
# "end"
|
12
|
-
# [Footer Part]
|
13
|
-
|
14
1
|
class OedipusLex
|
15
2
|
|
16
3
|
option
|
@@ -51,7 +38,11 @@ rule
|
|
51
38
|
|
52
39
|
:macro /\s+(\w+)\s+#{RE}/o { [:macro, *matches] }
|
53
40
|
|
54
|
-
:rule /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o
|
41
|
+
:rule /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:rule, *matches] }
|
42
|
+
:rule /\s*:[\ \t]*#{RE}/o { [:grouphead, *matches] }
|
43
|
+
:group /\s*:[\ \t]*#{RE}/o { [:grouphead, *matches] }
|
44
|
+
:group /\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:group, *matches] }
|
45
|
+
:group /\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o { [:groupend, *matches] }
|
55
46
|
|
56
47
|
:END /\n+/ # do nothing
|
57
48
|
:END /.*/ { [:end, text] }
|
data/lib/oedipus_lex.rex.rb
CHANGED
@@ -1,22 +1,9 @@
|
|
1
1
|
#--
|
2
2
|
# This file is automatically generated. Do not modify it.
|
3
|
-
# Generated by: oedipus_lex version 2.
|
3
|
+
# Generated by: oedipus_lex version 2.1.0.
|
4
4
|
# Source: lib/oedipus_lex.rex
|
5
5
|
#++
|
6
6
|
|
7
|
-
# [Header Part]
|
8
|
-
# "class" Foo
|
9
|
-
# ["option"
|
10
|
-
# [options] ]
|
11
|
-
# ["inner"
|
12
|
-
# [methods] ]
|
13
|
-
# ["macro"
|
14
|
-
# [macro-name /pattern/[flags]] ]
|
15
|
-
# "rule"
|
16
|
-
# [:state | method_name] /pattern/[flags] [{ code } | method_name | :state]
|
17
|
-
# "end"
|
18
|
-
# [Footer Part]
|
19
|
-
|
20
7
|
class OedipusLex
|
21
8
|
require 'strscan'
|
22
9
|
|
@@ -78,7 +65,7 @@ class OedipusLex
|
|
78
65
|
until ss.eos? or token do
|
79
66
|
token =
|
80
67
|
case state
|
81
|
-
when nil, :option, :inner, :start, :macro, :rule then
|
68
|
+
when nil, :option, :inner, :start, :macro, :rule, :group then
|
82
69
|
case
|
83
70
|
when text = ss.scan(/options?.*/) then
|
84
71
|
[:state, :option]
|
@@ -116,6 +103,14 @@ class OedipusLex
|
|
116
103
|
action { [:macro, *matches] }
|
117
104
|
when (state == :rule) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
|
118
105
|
action { [:rule, *matches] }
|
106
|
+
when (state == :rule) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
|
107
|
+
action { [:grouphead, *matches] }
|
108
|
+
when (state == :group) && (text = ss.scan(/\s*:[\ \t]*#{RE}/o)) then
|
109
|
+
action { [:grouphead, *matches] }
|
110
|
+
when (state == :group) && (text = ss.scan(/\s*\|\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
|
111
|
+
action { [:group, *matches] }
|
112
|
+
when (state == :group) && (text = ss.scan(/\s*#{ST}?[\ \t]*#{RE}[\ \t]*#{ACT}?/o)) then
|
113
|
+
action { [:groupend, *matches] }
|
119
114
|
else
|
120
115
|
text = ss.string[ss.pos .. -1]
|
121
116
|
raise ScanError, "can not match (#{state.inspect}): '#{text}'"
|
data/test/test_oedipus_lex.rb
CHANGED
@@ -420,6 +420,54 @@ class TestOedipusLex < Minitest::Test
|
|
420
420
|
assert_match "when :ARG then", ruby
|
421
421
|
end
|
422
422
|
|
423
|
+
def test_simple_scanner_group
|
424
|
+
src = <<-'REX'
|
425
|
+
class Calculator
|
426
|
+
rules
|
427
|
+
|
428
|
+
: /\d/
|
429
|
+
| /\d+\.\d+/ { [:float, text.to_f] }
|
430
|
+
| /\d+/ { [:int, text.to_i] }
|
431
|
+
/\s+/
|
432
|
+
end
|
433
|
+
REX
|
434
|
+
|
435
|
+
ruby = generate_lexer src
|
436
|
+
|
437
|
+
assert_match "when ss.check(/\\d/) then", ruby
|
438
|
+
assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
|
439
|
+
assert_match "when text = ss.scan(/\\d+/) then", ruby
|
440
|
+
assert_match "end # group /\\d/", ruby
|
441
|
+
end
|
442
|
+
|
443
|
+
def test_simple_scanner_group_I_am_dumb
|
444
|
+
src = <<-'REX'
|
445
|
+
class Calculator
|
446
|
+
rules
|
447
|
+
|
448
|
+
: /\d/
|
449
|
+
| /\d+\.\d+/ { [:float, text.to_f] }
|
450
|
+
| /\d+/ { [:int, text.to_i] }
|
451
|
+
: /\+/
|
452
|
+
| xx? /\+whatever/ { [:x, text] }
|
453
|
+
| :x /\+\d+/ { [:y, text] }
|
454
|
+
/\s+/
|
455
|
+
end
|
456
|
+
REX
|
457
|
+
|
458
|
+
ruby = generate_lexer src
|
459
|
+
|
460
|
+
assert_match "when ss.check(/\\d/) then", ruby
|
461
|
+
assert_match "when text = ss.scan(/\\d+\\.\\d+/) then", ruby
|
462
|
+
assert_match "when text = ss.scan(/\\d+/) then", ruby
|
463
|
+
assert_match "end # group /\\d/", ruby
|
464
|
+
|
465
|
+
assert_match "when ss.check(/\\+/) then", ruby
|
466
|
+
assert_match "when xx? && (text = ss.scan(/\\+whatever/)) then", ruby
|
467
|
+
assert_match "when (state == :x) && (text = ss.scan(/\\+\\d+/)) then", ruby
|
468
|
+
assert_match "end # group /\\d/", ruby
|
469
|
+
end
|
470
|
+
|
423
471
|
def test_generator_start
|
424
472
|
src = <<-'REX'
|
425
473
|
class Calculator
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: oedipus_lex
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Davis
|
@@ -29,7 +29,7 @@ cert_chain:
|
|
29
29
|
Y4evBVezr3SjXz08vPqRO5YRdO3zfeMT8gBjRqZjWJGMZ2lD4XNfrs7eky74CyZw
|
30
30
|
xx3n58i0lQkBE1EpKE0lFu/y
|
31
31
|
-----END CERTIFICATE-----
|
32
|
-
date: 2014-
|
32
|
+
date: 2014-03-14 00:00:00.000000000 Z
|
33
33
|
dependencies:
|
34
34
|
- !ruby/object:Gem::Dependency
|
35
35
|
name: minitest
|
@@ -37,14 +37,14 @@ dependencies:
|
|
37
37
|
requirements:
|
38
38
|
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '5.
|
40
|
+
version: '5.3'
|
41
41
|
type: :development
|
42
42
|
prerelease: false
|
43
43
|
version_requirements: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
45
|
- - ~>
|
46
46
|
- !ruby/object:Gem::Version
|
47
|
-
version: '5.
|
47
|
+
version: '5.3'
|
48
48
|
- !ruby/object:Gem::Dependency
|
49
49
|
name: rdoc
|
50
50
|
requirement: !ruby/object:Gem::Requirement
|
@@ -65,14 +65,14 @@ dependencies:
|
|
65
65
|
requirements:
|
66
66
|
- - ~>
|
67
67
|
- !ruby/object:Gem::Version
|
68
|
-
version: '3.
|
68
|
+
version: '3.10'
|
69
69
|
type: :development
|
70
70
|
prerelease: false
|
71
71
|
version_requirements: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
73
|
- - ~>
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '3.
|
75
|
+
version: '3.10'
|
76
76
|
description: |-
|
77
77
|
Oedipus Lex is a lexer generator in the same family as Rexical and
|
78
78
|
Rex. Oedipus Lex is my independent lexer fork of Rexical. Rexical was
|
@@ -107,7 +107,6 @@ extra_rdoc_files:
|
|
107
107
|
- sample/error1.txt
|
108
108
|
files:
|
109
109
|
- .autotest
|
110
|
-
- .gemtest
|
111
110
|
- History.rdoc
|
112
111
|
- Manifest.txt
|
113
112
|
- README.rdoc
|
@@ -134,6 +133,7 @@ files:
|
|
134
133
|
- sample/xhtmlparser.rex
|
135
134
|
- sample/xhtmlparser.xhtml
|
136
135
|
- test/test_oedipus_lex.rb
|
136
|
+
- .gemtest
|
137
137
|
homepage: http://github.com/seattlerb/oedipus_lex
|
138
138
|
licenses:
|
139
139
|
- MIT
|
@@ -155,8 +155,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
155
155
|
- !ruby/object:Gem::Version
|
156
156
|
version: '0'
|
157
157
|
requirements: []
|
158
|
-
rubyforge_project:
|
159
|
-
rubygems_version: 2.
|
158
|
+
rubyforge_project:
|
159
|
+
rubygems_version: 2.0.14
|
160
160
|
signing_key:
|
161
161
|
specification_version: 4
|
162
162
|
summary: Oedipus Lex is a lexer generator in the same family as Rexical and Rex
|
metadata.gz.sig
CHANGED
Binary file
|