rley 0.8.00 → 0.8.05
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +47 -3
- data/CHANGELOG.md +32 -4
- data/examples/NLP/pico_en_demo.rb +2 -2
- data/examples/data_formats/JSON/README.md +34 -0
- data/examples/data_formats/JSON/sample01.json +3 -0
- data/examples/data_formats/JSON/sample01.svg +36 -0
- data/examples/data_formats/JSON/sample02.json +6 -0
- data/examples/data_formats/JSON/sample02.svg +128 -0
- data/examples/data_formats/JSON/sample03.json +88 -0
- data/examples/general/calc_iter1/README.md +26 -0
- data/examples/general/calc_iter2/README.md +55 -0
- data/examples/general/general_examples.md +37 -0
- data/examples/tokenizer/README.md +46 -0
- data/examples/tokenizer/loxxy_raw_scanner.rex +98 -0
- data/examples/tokenizer/loxxy_raw_scanner.rex.rb +256 -0
- data/examples/tokenizer/loxxy_tokenizer.rb +94 -0
- data/examples/tokenizer/run_tokenizer.rb +29 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/lexical/literal.rb +29 -0
- data/lib/rley/lexical/token.rb +7 -4
- data/lib/rley/notation/all_notation_nodes.rb +3 -1
- data/lib/rley/notation/ast_builder.rb +185 -191
- data/lib/rley/notation/ast_node.rb +5 -5
- data/lib/rley/notation/ast_visitor.rb +3 -1
- data/lib/rley/notation/grammar.rb +1 -1
- data/lib/rley/notation/grammar_builder.rb +87 -33
- data/lib/rley/notation/grouping_node.rb +1 -1
- data/lib/rley/notation/parser.rb +56 -56
- data/lib/rley/notation/sequence_node.rb +3 -3
- data/lib/rley/notation/symbol_node.rb +2 -2
- data/lib/rley/notation/tokenizer.rb +3 -15
- data/lib/rley/parse_rep/ast_base_builder.rb +35 -4
- data/lib/rley/parser/gfg_chart.rb +5 -4
- data/lib/rley/parser/gfg_earley_parser.rb +1 -1
- data/lib/rley/syntax/base_grammar_builder.rb +8 -2
- data/lib/rley/syntax/match_closest.rb +7 -7
- data/lib/rley.rb +1 -1
- data/spec/rley/lexical/literal_spec.rb +33 -0
- data/spec/rley/lexical/token_spec.rb +15 -4
- data/spec/rley/notation/grammar_builder_spec.rb +57 -50
- data/spec/rley/notation/parser_spec.rb +183 -184
- data/spec/rley/notation/tokenizer_spec.rb +98 -104
- data/spec/rley/parser/dangling_else_spec.rb +20 -20
- data/spec/rley/parser/gfg_chart_spec.rb +0 -1
- data/spec/rley/parser/gfg_earley_parser_spec.rb +166 -147
- data/spec/rley/parser/gfg_parsing_spec.rb +2 -2
- data/spec/rley/syntax/base_grammar_builder_spec.rb +7 -8
- data/spec/rley/syntax/grammar_spec.rb +6 -9
- data/spec/rley/syntax/match_closest_spec.rb +4 -4
- metadata +19 -9
- data/lib/rley/parser/parse_tracer.rb +0 -103
- data/lib/rley/syntax/literal.rb +0 -20
- data/lib/rley/syntax/verbatim_symbol.rb +0 -27
- data/spec/rley/syntax/literal_spec.rb +0 -31
- data/spec/rley/syntax/verbatim_symbol_spec.rb +0 -38
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
|
5
|
+
require_relative '../../../lib/rley/syntax/terminal'
|
6
|
+
|
7
|
+
# Load the class under test
|
8
|
+
require_relative '../../../lib/rley/lexical/literal'
|
9
|
+
|
10
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
11
|
+
module Lexical # Open this namespace to avoid module qualifier prefixes
|
12
|
+
describe Literal do
|
13
|
+
let(:lexeme) { '12.34' }
|
14
|
+
let(:a_terminal) { Syntax::Terminal.new('NUMBER') }
|
15
|
+
let(:a_pos) { Position.new(3, 4) }
|
16
|
+
|
17
|
+
context 'Initialization:' do
|
18
|
+
# Default instantiation rule
|
19
|
+
subject { Literal.new(lexeme.to_f, lexeme, a_terminal, a_pos) }
|
20
|
+
|
21
|
+
it 'should be created with a value, lexeme, terminal and position' do
|
22
|
+
expect { Literal.new(lexeme.to_f, lexeme, a_terminal, a_pos) }.not_to raise_error
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should know its value' do
|
26
|
+
expect(subject.value).to eq(lexeme.to_f)
|
27
|
+
end
|
28
|
+
end # context
|
29
|
+
end # describe
|
30
|
+
end # module
|
31
|
+
end # module
|
32
|
+
|
33
|
+
# End of file
|
@@ -13,12 +13,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
13
13
|
let(:lexeme) { '"some text"' }
|
14
14
|
let(:a_terminal) { Syntax::Terminal.new('if') }
|
15
15
|
let(:a_pos) { Position.new(3, 4) }
|
16
|
+
# Default instantiation rule
|
17
|
+
subject { Token.new(lexeme, a_terminal, a_pos) }
|
16
18
|
|
17
19
|
context 'Initialization:' do
|
18
|
-
|
19
|
-
|
20
|
+
it 'could be created with a lexeme and a terminal ' do
|
21
|
+
expect { Token.new(lexeme, a_terminal) }.not_to raise_error
|
22
|
+
end
|
20
23
|
|
21
|
-
it '
|
24
|
+
it 'could be created with a lexeme, a terminal and position' do
|
22
25
|
expect { Token.new(lexeme, a_terminal, a_pos) }.not_to raise_error
|
23
26
|
end
|
24
27
|
|
@@ -30,7 +33,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
30
33
|
expect(subject.terminal).to eq(a_terminal)
|
31
34
|
end
|
32
35
|
|
33
|
-
it 'should know its
|
36
|
+
it 'should know its position' do
|
37
|
+
new_pos = Position.new(5, 7)
|
38
|
+
subject.position = new_pos
|
39
|
+
expect(subject.position).to eq(new_pos)
|
40
|
+
end
|
41
|
+
end # context
|
42
|
+
|
43
|
+
context 'Initialization:' do
|
44
|
+
it 'should accept a new position' do
|
34
45
|
expect(subject.position).to eq(a_pos)
|
35
46
|
end
|
36
47
|
end # context
|
@@ -55,8 +55,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
55
55
|
|
56
56
|
it 'should accept already built terminals' do
|
57
57
|
a = Syntax::Terminal.new('a')
|
58
|
-
b = Syntax::
|
59
|
-
c = Syntax::
|
58
|
+
b = Syntax::Terminal.new('b')
|
59
|
+
c = Syntax::Terminal.new('c')
|
60
60
|
|
61
61
|
subject.add_terminals(a, b, c)
|
62
62
|
expect(subject.symbols.size).to eq(3)
|
@@ -113,70 +113,78 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
113
113
|
expect(new_prod.constraints[0].closest_symb).to eq('IF')
|
114
114
|
end
|
115
115
|
|
116
|
-
it
|
116
|
+
it 'should support optional symbol' do
|
117
117
|
instance = GrammarBuilder.new
|
118
118
|
instance.add_terminals('LPAREN', 'RPAREN')
|
119
119
|
|
120
|
-
instance.
|
120
|
+
instance.rule 'argument_list' => 'LPAREN arguments? RPAREN'
|
121
|
+
instance.grammar_complete!
|
121
122
|
|
122
123
|
# implicitly called: rule('arguments_qmark' => 'arguments').tag suffix_qmark_one
|
123
124
|
# implicitly called: rule('arguments_qmark' => '').tag suffix_qmark_none
|
124
125
|
expect(instance.productions.size).to eq(3)
|
125
126
|
prod_star = instance.productions.select { |prod| prod.lhs.name == 'arguments_qmark' }
|
126
127
|
expect(prod_star.size).to eq(2)
|
127
|
-
|
128
|
-
expect(
|
129
|
-
expect(
|
128
|
+
first_prod = instance.productions.first
|
129
|
+
expect(first_prod.lhs.name).to eq('argument_list')
|
130
|
+
expect(first_prod.rhs.members[1].name).to eq('arguments_qmark')
|
130
131
|
end
|
131
132
|
|
132
133
|
it "should support Kleene's star" do
|
133
134
|
instance = GrammarBuilder.new
|
134
135
|
instance.add_terminals('EOF')
|
135
136
|
|
136
|
-
instance.
|
137
|
+
instance.rule 'program' => 'declaration* EOF'
|
138
|
+
instance.grammar_complete!
|
137
139
|
|
138
140
|
# implicitly called: rule('declaration_star' => 'declaration_star declaration').tag suffix_star_more
|
139
141
|
# implicitly called: rule('declaration_star' => '').tag suffix_star_last
|
140
142
|
expect(instance.productions.size).to eq(3)
|
141
143
|
prod_star = instance.productions.select { |prod| prod.lhs.name == 'declaration_star' }
|
142
144
|
expect(prod_star.size).to eq(2)
|
143
|
-
|
144
|
-
expect(
|
145
|
-
expect(
|
145
|
+
first_prod = instance.productions.first
|
146
|
+
expect(first_prod.lhs.name).to eq('program')
|
147
|
+
expect(first_prod.rhs.members[0].name).to eq('declaration_star')
|
146
148
|
end
|
147
149
|
|
148
150
|
it "should support symbols decorated with Kleene's plus" do
|
149
151
|
instance = GrammarBuilder.new
|
150
152
|
instance.add_terminals('plus', 'minus', 'digit')
|
151
153
|
|
152
|
-
instance.
|
153
|
-
instance.
|
154
|
-
instance.
|
155
|
-
instance.
|
156
|
-
|
157
|
-
instance.
|
154
|
+
instance.rule 'integer' => 'value'
|
155
|
+
instance.rule 'integer' => 'sign value'
|
156
|
+
instance.rule 'sign' => 'plus'
|
157
|
+
instance.rule 'sign' => 'minus'
|
158
|
+
instance.rule 'value' => 'digit+'
|
159
|
+
expect(instance.productions.size).to eq(5)
|
160
|
+
instance.grammar_complete!
|
161
|
+
expect(instance.productions.size).to eq(7)
|
158
162
|
|
159
163
|
# implicitly called: rule('digit_plus' => 'digit_plus digit').tag suffix_plus_more
|
160
164
|
# implicitly called: rule('digit_plus' => 'digit').tag suffix_plus_last
|
161
165
|
expect(instance.productions.size).to eq(7) # Two additional rules generated
|
162
166
|
prod_plus = instance.productions.select { |prod| prod.lhs.name == 'digit_plus' }
|
163
167
|
expect(prod_plus.size).to eq(2)
|
164
|
-
|
165
|
-
expect(
|
166
|
-
expect(
|
168
|
+
val_prod = instance.productions[4]
|
169
|
+
expect(val_prod.lhs.name).to eq('value')
|
170
|
+
expect(val_prod.rhs.members[0].name).to eq('digit_plus')
|
167
171
|
end
|
168
172
|
|
169
|
-
it
|
173
|
+
it 'should support optional grouping' do
|
170
174
|
instance = GrammarBuilder.new
|
171
175
|
instance.add_terminals('EQUAL', 'IDENTIFIER', 'VAR')
|
172
176
|
|
173
|
-
instance.
|
177
|
+
instance.rule 'var_decl' => 'VAR IDENTIFIER (EQUAL expression)?'
|
178
|
+
instance.grammar_complete!
|
174
179
|
|
175
180
|
# implicitly called: rule('seq_EQUAL_expression' => 'EQUAL expression').tag 'return_children'
|
176
181
|
# implicitly called: rule('seq_EQUAL_expression_qmark' => 'seq_EQUAL_expression').tag suffix_qmark_one
|
177
182
|
# implicitly called: rule('seq_EQUAL_expression_qmark' => '').tag suffix_qmark_none
|
178
183
|
expect(instance.productions.size).to eq(4)
|
179
|
-
|
184
|
+
first_prod = instance.productions.first
|
185
|
+
expect(first_prod.lhs.name).to eq('var_decl')
|
186
|
+
expect(first_prod.rhs.members[2].name).to eq('seq_EQUAL_expression_qmark')
|
187
|
+
(p0, p1, p2) = instance.productions[1..3]
|
180
188
|
expect(p0.lhs.name).to eq('seq_EQUAL_expression')
|
181
189
|
expect(p0.rhs[0].name).to eq('EQUAL')
|
182
190
|
expect(p0.rhs[1].name).to eq('expression')
|
@@ -189,23 +197,24 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
189
197
|
expect(p2.lhs.name).to eq('seq_EQUAL_expression_qmark')
|
190
198
|
expect(p2.rhs).to be_empty
|
191
199
|
expect(p2.name).to eq('_qmark_none')
|
192
|
-
|
193
|
-
last_prod = instance.productions.last
|
194
|
-
expect(last_prod.lhs.name).to eq('var_decl')
|
195
|
-
expect(last_prod.rhs.members[2].name).to eq('seq_EQUAL_expression_qmark')
|
196
200
|
end
|
197
201
|
|
198
|
-
it
|
202
|
+
it 'should support grouping with star modifier' do
|
199
203
|
instance = GrammarBuilder.new
|
200
204
|
instance.add_terminals('OR')
|
201
205
|
|
202
|
-
instance.
|
206
|
+
instance.rule 'logic_or' => 'logic_and (OR logic_and)*'
|
207
|
+
instance.grammar_complete!
|
203
208
|
|
204
209
|
# implicitly called: rule('seq_OR_logic_and' => 'OR logic_and').tag 'return_children'
|
205
210
|
# implicitly called: rule('seq_EQUAL_expression_star' => 'seq_EQUAL_expression_star seq_EQUAL_expression').tag suffix_star_more
|
206
211
|
# implicitly called: rule('seq_EQUAL_expression_star' => '').tag suffix_star_none
|
207
212
|
expect(instance.productions.size).to eq(4)
|
208
|
-
|
213
|
+
first_prod = instance.productions.first
|
214
|
+
expect(first_prod.lhs.name).to eq('logic_or')
|
215
|
+
expect(first_prod.rhs.members[1].name).to eq('seq_OR_logic_and_star')
|
216
|
+
|
217
|
+
(p0, p1, p2) = instance.productions[1..3]
|
209
218
|
expect(p0.lhs.name).to eq('seq_OR_logic_and')
|
210
219
|
expect(p0.rhs[0].name).to eq('OR')
|
211
220
|
expect(p0.rhs[1].name).to eq('logic_and')
|
@@ -219,23 +228,24 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
219
228
|
expect(p2.lhs.name).to eq('seq_OR_logic_and_star')
|
220
229
|
expect(p2.rhs).to be_empty
|
221
230
|
expect(p2.name).to eq('_star_none')
|
222
|
-
|
223
|
-
last_prod = instance.productions.last
|
224
|
-
expect(last_prod.lhs.name).to eq('logic_or')
|
225
|
-
expect(last_prod.rhs.members[1].name).to eq('seq_OR_logic_and_star')
|
226
231
|
end
|
227
232
|
|
228
|
-
it
|
233
|
+
it 'should support grouping with plus modifier' do
|
229
234
|
instance = GrammarBuilder.new
|
230
235
|
instance.add_terminals('POINT TO SEMI_COLON')
|
231
236
|
|
232
|
-
instance.
|
237
|
+
instance.rule 'path' => 'POINT (TO POINT)+ SEMI_COLON'
|
238
|
+
instance.grammar_complete!
|
233
239
|
|
234
240
|
# implicitly called: rule('seq_TO_POINT' => 'TO POINT').tag 'return_children'
|
235
241
|
# implicitly called: rule('seq_TO_POINT_plus' => 'seq_TO_POINT_plus seq_TO_POINT').tag suffix_plus_more
|
236
242
|
# implicitly called: rule('seq_TO_POINT_plus' => 'seq_TO_POINT').tag suffix_plus_one
|
237
243
|
expect(instance.productions.size).to eq(4)
|
238
|
-
|
244
|
+
first_prod = instance.productions.first
|
245
|
+
expect(first_prod.lhs.name).to eq('path')
|
246
|
+
expect(first_prod.rhs.members[1].name).to eq('seq_TO_POINT_plus')
|
247
|
+
|
248
|
+
(p0, p1, p2) = instance.productions[1..3]
|
239
249
|
expect(p0.lhs.name).to eq('seq_TO_POINT')
|
240
250
|
expect(p0.rhs[0].name).to eq('TO')
|
241
251
|
expect(p0.rhs[1].name).to eq('POINT')
|
@@ -249,28 +259,29 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
249
259
|
expect(p2.lhs.name).to eq('seq_TO_POINT_plus')
|
250
260
|
expect(p2.rhs[0].name).to eq('seq_TO_POINT')
|
251
261
|
expect(p2.name).to eq('_plus_one')
|
252
|
-
|
253
|
-
last_prod = instance.productions.last
|
254
|
-
expect(last_prod.lhs.name).to eq('path')
|
255
|
-
expect(last_prod.rhs.members[1].name).to eq('seq_TO_POINT_plus')
|
256
262
|
end
|
257
263
|
|
258
|
-
it
|
264
|
+
it 'should support grouping with nested annotation' do
|
259
265
|
instance = GrammarBuilder.new
|
260
266
|
instance.add_terminals('IF ELSE LPAREN RPAREN')
|
261
267
|
st = "IF LPAREN expr RPAREN stmt (ELSE { match_closest: 'IF' } stmt)?"
|
262
|
-
instance.
|
268
|
+
instance.rule('if_stmt' => st)
|
269
|
+
instance.grammar_complete!
|
263
270
|
|
264
271
|
# implicitly called: rule('seq_ELSE_stmt' => 'ELSE stmt').tag 'return_children'
|
265
272
|
# implicitly called: rule('seq_ELSE_stmt_qmark' => 'seq_ELSE_stmt ').tag suffix_plus_more
|
266
273
|
# implicitly called: rule('seq_ELSE_stmt_qmark' => '').tag suffix_plus_one
|
267
274
|
expect(instance.productions.size).to eq(4)
|
268
|
-
|
275
|
+
first_prod = instance.productions.first
|
276
|
+
expect(first_prod.lhs.name).to eq('if_stmt')
|
277
|
+
expect(first_prod.rhs.members[5].name).to eq('seq_ELSE_stmt_qmark')
|
278
|
+
|
279
|
+
(p0, p1, p2) = instance.productions[1..3]
|
269
280
|
expect(p0.lhs.name).to eq('seq_ELSE_stmt')
|
270
281
|
expect(p0.rhs[0].name).to eq('ELSE')
|
271
282
|
expect(p0.rhs[1].name).to eq('stmt')
|
272
283
|
expect(p0.name).to eq('return_children')
|
273
|
-
expect(p0.constraints.size).
|
284
|
+
expect(p0.constraints.size).to eq(1)
|
274
285
|
expect(p0.constraints[0]).to be_kind_of(Syntax::MatchClosest)
|
275
286
|
expect(p0.constraints[0].idx_symbol).to eq(0) # ELSE is on position 0
|
276
287
|
expect(p0.constraints[0].closest_symb).to eq('IF')
|
@@ -282,14 +293,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
282
293
|
expect(p2.lhs.name).to eq('seq_ELSE_stmt_qmark')
|
283
294
|
expect(p2.rhs).to be_empty
|
284
295
|
expect(p2.name).to eq('_qmark_none')
|
285
|
-
|
286
|
-
last_prod = instance.productions.last
|
287
|
-
expect(last_prod.lhs.name).to eq('if_stmt')
|
288
|
-
expect(last_prod.rhs.members[5].name).to eq('seq_ELSE_stmt_qmark')
|
289
296
|
end
|
290
297
|
end # context
|
291
298
|
end # describe
|
292
299
|
end # module
|
293
300
|
end # module
|
294
301
|
|
295
|
-
# End of file
|
302
|
+
# End of file
|