rley 0.8.00 → 0.8.05

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +47 -3
  3. data/CHANGELOG.md +32 -4
  4. data/examples/NLP/pico_en_demo.rb +2 -2
  5. data/examples/data_formats/JSON/README.md +34 -0
  6. data/examples/data_formats/JSON/sample01.json +3 -0
  7. data/examples/data_formats/JSON/sample01.svg +36 -0
  8. data/examples/data_formats/JSON/sample02.json +6 -0
  9. data/examples/data_formats/JSON/sample02.svg +128 -0
  10. data/examples/data_formats/JSON/sample03.json +88 -0
  11. data/examples/general/calc_iter1/README.md +26 -0
  12. data/examples/general/calc_iter2/README.md +55 -0
  13. data/examples/general/general_examples.md +37 -0
  14. data/examples/tokenizer/README.md +46 -0
  15. data/examples/tokenizer/loxxy_raw_scanner.rex +98 -0
  16. data/examples/tokenizer/loxxy_raw_scanner.rex.rb +256 -0
  17. data/examples/tokenizer/loxxy_tokenizer.rb +94 -0
  18. data/examples/tokenizer/run_tokenizer.rb +29 -0
  19. data/lib/rley/constants.rb +1 -1
  20. data/lib/rley/lexical/literal.rb +29 -0
  21. data/lib/rley/lexical/token.rb +7 -4
  22. data/lib/rley/notation/all_notation_nodes.rb +3 -1
  23. data/lib/rley/notation/ast_builder.rb +185 -191
  24. data/lib/rley/notation/ast_node.rb +5 -5
  25. data/lib/rley/notation/ast_visitor.rb +3 -1
  26. data/lib/rley/notation/grammar.rb +1 -1
  27. data/lib/rley/notation/grammar_builder.rb +87 -33
  28. data/lib/rley/notation/grouping_node.rb +1 -1
  29. data/lib/rley/notation/parser.rb +56 -56
  30. data/lib/rley/notation/sequence_node.rb +3 -3
  31. data/lib/rley/notation/symbol_node.rb +2 -2
  32. data/lib/rley/notation/tokenizer.rb +3 -15
  33. data/lib/rley/parse_rep/ast_base_builder.rb +35 -4
  34. data/lib/rley/parser/gfg_chart.rb +5 -4
  35. data/lib/rley/parser/gfg_earley_parser.rb +1 -1
  36. data/lib/rley/syntax/base_grammar_builder.rb +8 -2
  37. data/lib/rley/syntax/match_closest.rb +7 -7
  38. data/lib/rley.rb +1 -1
  39. data/spec/rley/lexical/literal_spec.rb +33 -0
  40. data/spec/rley/lexical/token_spec.rb +15 -4
  41. data/spec/rley/notation/grammar_builder_spec.rb +57 -50
  42. data/spec/rley/notation/parser_spec.rb +183 -184
  43. data/spec/rley/notation/tokenizer_spec.rb +98 -104
  44. data/spec/rley/parser/dangling_else_spec.rb +20 -20
  45. data/spec/rley/parser/gfg_chart_spec.rb +0 -1
  46. data/spec/rley/parser/gfg_earley_parser_spec.rb +166 -147
  47. data/spec/rley/parser/gfg_parsing_spec.rb +2 -2
  48. data/spec/rley/syntax/base_grammar_builder_spec.rb +7 -8
  49. data/spec/rley/syntax/grammar_spec.rb +6 -9
  50. data/spec/rley/syntax/match_closest_spec.rb +4 -4
  51. metadata +19 -9
  52. data/lib/rley/parser/parse_tracer.rb +0 -103
  53. data/lib/rley/syntax/literal.rb +0 -20
  54. data/lib/rley/syntax/verbatim_symbol.rb +0 -27
  55. data/spec/rley/syntax/literal_spec.rb +0 -31
  56. data/spec/rley/syntax/verbatim_symbol_spec.rb +0 -38
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../spec_helper'
4
+
5
+ require_relative '../../../lib/rley/syntax/terminal'
6
+
7
+ # Load the class under test
8
+ require_relative '../../../lib/rley/lexical/literal'
9
+
10
+ module Rley # Open this namespace to avoid module qualifier prefixes
11
+ module Lexical # Open this namespace to avoid module qualifier prefixes
12
+ describe Literal do
13
+ let(:lexeme) { '12.34' }
14
+ let(:a_terminal) { Syntax::Terminal.new('NUMBER') }
15
+ let(:a_pos) { Position.new(3, 4) }
16
+
17
+ context 'Initialization:' do
18
+ # Default instantiation rule
19
+ subject { Literal.new(lexeme.to_f, lexeme, a_terminal, a_pos) }
20
+
21
+ it 'should be created with a value, lexeme, terminal and position' do
22
+ expect { Literal.new(lexeme.to_f, lexeme, a_terminal, a_pos) }.not_to raise_error
23
+ end
24
+
25
+ it 'should know its value' do
26
+ expect(subject.value).to eq(lexeme.to_f)
27
+ end
28
+ end # context
29
+ end # describe
30
+ end # module
31
+ end # module
32
+
33
+ # End of file
@@ -13,12 +13,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
13
13
  let(:lexeme) { '"some text"' }
14
14
  let(:a_terminal) { Syntax::Terminal.new('if') }
15
15
  let(:a_pos) { Position.new(3, 4) }
16
+ # Default instantiation rule
17
+ subject { Token.new(lexeme, a_terminal, a_pos) }
16
18
 
17
19
  context 'Initialization:' do
18
- # Default instantiation rule
19
- subject { Token.new(lexeme, a_terminal, a_pos) }
20
+ it 'could be created with a lexeme and a terminal ' do
21
+ expect { Token.new(lexeme, a_terminal) }.not_to raise_error
22
+ end
20
23
 
21
- it 'should be created with a lexeme, a terminal and position' do
24
+ it 'could be created with a lexeme, a terminal and position' do
22
25
  expect { Token.new(lexeme, a_terminal, a_pos) }.not_to raise_error
23
26
  end
24
27
 
@@ -30,7 +33,15 @@ module Rley # Open this namespace to avoid module qualifier prefixes
30
33
  expect(subject.terminal).to eq(a_terminal)
31
34
  end
32
35
 
33
- it 'should know its terminal' do
36
+ it 'should know its position' do
37
+ new_pos = Position.new(5, 7)
38
+ subject.position = new_pos
39
+ expect(subject.position).to eq(new_pos)
40
+ end
41
+ end # context
42
+
43
+ context 'Initialization:' do
44
+ it 'should accept a new position' do
34
45
  expect(subject.position).to eq(a_pos)
35
46
  end
36
47
  end # context
@@ -55,8 +55,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
55
55
 
56
56
  it 'should accept already built terminals' do
57
57
  a = Syntax::Terminal.new('a')
58
- b = Syntax::VerbatimSymbol.new('b')
59
- c = Syntax::Literal.new('c', /c/)
58
+ b = Syntax::Terminal.new('b')
59
+ c = Syntax::Terminal.new('c')
60
60
 
61
61
  subject.add_terminals(a, b, c)
62
62
  expect(subject.symbols.size).to eq(3)
@@ -113,70 +113,78 @@ module Rley # Open this namespace to avoid module qualifier prefixes
113
113
  expect(new_prod.constraints[0].closest_symb).to eq('IF')
114
114
  end
115
115
 
116
- it "should support optional symbol" do
116
+ it 'should support optional symbol' do
117
117
  instance = GrammarBuilder.new
118
118
  instance.add_terminals('LPAREN', 'RPAREN')
119
119
 
120
- instance.add_production('argument_list' => 'LPAREN arguments? RPAREN')
120
+ instance.rule 'argument_list' => 'LPAREN arguments? RPAREN'
121
+ instance.grammar_complete!
121
122
 
122
123
  # implicitly called: rule('arguments_qmark' => 'arguments').tag suffix_qmark_one
123
124
  # implicitly called: rule('arguments_qmark' => '').tag suffix_qmark_none
124
125
  expect(instance.productions.size).to eq(3)
125
126
  prod_star = instance.productions.select { |prod| prod.lhs.name == 'arguments_qmark' }
126
127
  expect(prod_star.size).to eq(2)
127
- last_prod = instance.productions.last
128
- expect(last_prod.lhs.name).to eq('argument_list')
129
- expect(last_prod.rhs.members[1].name).to eq('arguments_qmark')
128
+ first_prod = instance.productions.first
129
+ expect(first_prod.lhs.name).to eq('argument_list')
130
+ expect(first_prod.rhs.members[1].name).to eq('arguments_qmark')
130
131
  end
131
132
 
132
133
  it "should support Kleene's star" do
133
134
  instance = GrammarBuilder.new
134
135
  instance.add_terminals('EOF')
135
136
 
136
- instance.add_production('program' => 'declaration* EOF')
137
+ instance.rule 'program' => 'declaration* EOF'
138
+ instance.grammar_complete!
137
139
 
138
140
  # implicitly called: rule('declaration_star' => 'declaration_star declaration').tag suffix_star_more
139
141
  # implicitly called: rule('declaration_star' => '').tag suffix_star_last
140
142
  expect(instance.productions.size).to eq(3)
141
143
  prod_star = instance.productions.select { |prod| prod.lhs.name == 'declaration_star' }
142
144
  expect(prod_star.size).to eq(2)
143
- last_prod = instance.productions.last
144
- expect(last_prod.lhs.name).to eq('program')
145
- expect(last_prod.rhs.members[0].name).to eq('declaration_star')
145
+ first_prod = instance.productions.first
146
+ expect(first_prod.lhs.name).to eq('program')
147
+ expect(first_prod.rhs.members[0].name).to eq('declaration_star')
146
148
  end
147
149
 
148
150
  it "should support symbols decorated with Kleene's plus" do
149
151
  instance = GrammarBuilder.new
150
152
  instance.add_terminals('plus', 'minus', 'digit')
151
153
 
152
- instance.add_production('integer' => 'value')
153
- instance.add_production('integer' => 'sign value')
154
- instance.add_production('sign' => 'plus')
155
- instance.add_production('sign' => 'minus')
156
- expect(instance.productions.size).to eq(4)
157
- instance.add_production('value' => 'digit+')
154
+ instance.rule 'integer' => 'value'
155
+ instance.rule 'integer' => 'sign value'
156
+ instance.rule 'sign' => 'plus'
157
+ instance.rule 'sign' => 'minus'
158
+ instance.rule 'value' => 'digit+'
159
+ expect(instance.productions.size).to eq(5)
160
+ instance.grammar_complete!
161
+ expect(instance.productions.size).to eq(7)
158
162
 
159
163
  # implicitly called: rule('digit_plus' => 'digit_plus digit').tag suffix_plus_more
160
164
  # implicitly called: rule('digit_plus' => 'digit').tag suffix_plus_last
161
165
  expect(instance.productions.size).to eq(7) # Two additional rules generated
162
166
  prod_plus = instance.productions.select { |prod| prod.lhs.name == 'digit_plus' }
163
167
  expect(prod_plus.size).to eq(2)
164
- last_prod = instance.productions.last
165
- expect(last_prod.lhs.name).to eq('value')
166
- expect(last_prod.rhs.members[0].name).to eq('digit_plus')
168
+ val_prod = instance.productions[4]
169
+ expect(val_prod.lhs.name).to eq('value')
170
+ expect(val_prod.rhs.members[0].name).to eq('digit_plus')
167
171
  end
168
172
 
169
- it "should support optional grouping" do
173
+ it 'should support optional grouping' do
170
174
  instance = GrammarBuilder.new
171
175
  instance.add_terminals('EQUAL', 'IDENTIFIER', 'VAR')
172
176
 
173
- instance.add_production('var_decl' => 'VAR IDENTIFIER (EQUAL expression)?')
177
+ instance.rule 'var_decl' => 'VAR IDENTIFIER (EQUAL expression)?'
178
+ instance.grammar_complete!
174
179
 
175
180
  # implicitly called: rule('seq_EQUAL_expression' => 'EQUAL expression').tag 'return_children'
176
181
  # implicitly called: rule('seq_EQUAL_expression_qmark' => 'seq_EQUAL_expression').tag suffix_qmark_one
177
182
  # implicitly called: rule('seq_EQUAL_expression_qmark' => '').tag suffix_qmark_none
178
183
  expect(instance.productions.size).to eq(4)
179
- (p0, p1, p2) = instance.productions[0..2]
184
+ first_prod = instance.productions.first
185
+ expect(first_prod.lhs.name).to eq('var_decl')
186
+ expect(first_prod.rhs.members[2].name).to eq('seq_EQUAL_expression_qmark')
187
+ (p0, p1, p2) = instance.productions[1..3]
180
188
  expect(p0.lhs.name).to eq('seq_EQUAL_expression')
181
189
  expect(p0.rhs[0].name).to eq('EQUAL')
182
190
  expect(p0.rhs[1].name).to eq('expression')
@@ -189,23 +197,24 @@ module Rley # Open this namespace to avoid module qualifier prefixes
189
197
  expect(p2.lhs.name).to eq('seq_EQUAL_expression_qmark')
190
198
  expect(p2.rhs).to be_empty
191
199
  expect(p2.name).to eq('_qmark_none')
192
-
193
- last_prod = instance.productions.last
194
- expect(last_prod.lhs.name).to eq('var_decl')
195
- expect(last_prod.rhs.members[2].name).to eq('seq_EQUAL_expression_qmark')
196
200
  end
197
201
 
198
- it "should support grouping with star modifier" do
202
+ it 'should support grouping with star modifier' do
199
203
  instance = GrammarBuilder.new
200
204
  instance.add_terminals('OR')
201
205
 
202
- instance.add_production('logic_or' => 'logic_and (OR logic_and)*')
206
+ instance.rule 'logic_or' => 'logic_and (OR logic_and)*'
207
+ instance.grammar_complete!
203
208
 
204
209
  # implicitly called: rule('seq_OR_logic_and' => 'OR logic_and').tag 'return_children'
205
210
  # implicitly called: rule('seq_EQUAL_expression_star' => 'seq_EQUAL_expression_star seq_EQUAL_expression').tag suffix_star_more
206
211
  # implicitly called: rule('seq_EQUAL_expression_star' => '').tag suffix_star_none
207
212
  expect(instance.productions.size).to eq(4)
208
- (p0, p1, p2) = instance.productions[0..2]
213
+ first_prod = instance.productions.first
214
+ expect(first_prod.lhs.name).to eq('logic_or')
215
+ expect(first_prod.rhs.members[1].name).to eq('seq_OR_logic_and_star')
216
+
217
+ (p0, p1, p2) = instance.productions[1..3]
209
218
  expect(p0.lhs.name).to eq('seq_OR_logic_and')
210
219
  expect(p0.rhs[0].name).to eq('OR')
211
220
  expect(p0.rhs[1].name).to eq('logic_and')
@@ -219,23 +228,24 @@ module Rley # Open this namespace to avoid module qualifier prefixes
219
228
  expect(p2.lhs.name).to eq('seq_OR_logic_and_star')
220
229
  expect(p2.rhs).to be_empty
221
230
  expect(p2.name).to eq('_star_none')
222
-
223
- last_prod = instance.productions.last
224
- expect(last_prod.lhs.name).to eq('logic_or')
225
- expect(last_prod.rhs.members[1].name).to eq('seq_OR_logic_and_star')
226
231
  end
227
232
 
228
- it "should support grouping with plus modifier" do
233
+ it 'should support grouping with plus modifier' do
229
234
  instance = GrammarBuilder.new
230
235
  instance.add_terminals('POINT TO SEMI_COLON')
231
236
 
232
- instance.add_production('path' => 'POINT (TO POINT)+ SEMI_COLON')
237
+ instance.rule 'path' => 'POINT (TO POINT)+ SEMI_COLON'
238
+ instance.grammar_complete!
233
239
 
234
240
  # implicitly called: rule('seq_TO_POINT' => 'TO POINT').tag 'return_children'
235
241
  # implicitly called: rule('seq_TO_POINT_plus' => 'seq_TO_POINT_plus seq_TO_POINT').tag suffix_plus_more
236
242
  # implicitly called: rule('seq_TO_POINT_plus' => 'seq_TO_POINT').tag suffix_plus_one
237
243
  expect(instance.productions.size).to eq(4)
238
- (p0, p1, p2) = instance.productions[0..2]
244
+ first_prod = instance.productions.first
245
+ expect(first_prod.lhs.name).to eq('path')
246
+ expect(first_prod.rhs.members[1].name).to eq('seq_TO_POINT_plus')
247
+
248
+ (p0, p1, p2) = instance.productions[1..3]
239
249
  expect(p0.lhs.name).to eq('seq_TO_POINT')
240
250
  expect(p0.rhs[0].name).to eq('TO')
241
251
  expect(p0.rhs[1].name).to eq('POINT')
@@ -249,28 +259,29 @@ module Rley # Open this namespace to avoid module qualifier prefixes
249
259
  expect(p2.lhs.name).to eq('seq_TO_POINT_plus')
250
260
  expect(p2.rhs[0].name).to eq('seq_TO_POINT')
251
261
  expect(p2.name).to eq('_plus_one')
252
-
253
- last_prod = instance.productions.last
254
- expect(last_prod.lhs.name).to eq('path')
255
- expect(last_prod.rhs.members[1].name).to eq('seq_TO_POINT_plus')
256
262
  end
257
263
 
258
- it "should support grouping with nested annotation" do
264
+ it 'should support grouping with nested annotation' do
259
265
  instance = GrammarBuilder.new
260
266
  instance.add_terminals('IF ELSE LPAREN RPAREN')
261
267
  st = "IF LPAREN expr RPAREN stmt (ELSE { match_closest: 'IF' } stmt)?"
262
- instance.add_production('if_stmt' => st)
268
+ instance.rule('if_stmt' => st)
269
+ instance.grammar_complete!
263
270
 
264
271
  # implicitly called: rule('seq_ELSE_stmt' => 'ELSE stmt').tag 'return_children'
265
272
  # implicitly called: rule('seq_ELSE_stmt_qmark' => 'seq_ELSE_stmt ').tag suffix_plus_more
266
273
  # implicitly called: rule('seq_ELSE_stmt_qmark' => '').tag suffix_plus_one
267
274
  expect(instance.productions.size).to eq(4)
268
- (p0, p1, p2) = instance.productions[0..2]
275
+ first_prod = instance.productions.first
276
+ expect(first_prod.lhs.name).to eq('if_stmt')
277
+ expect(first_prod.rhs.members[5].name).to eq('seq_ELSE_stmt_qmark')
278
+
279
+ (p0, p1, p2) = instance.productions[1..3]
269
280
  expect(p0.lhs.name).to eq('seq_ELSE_stmt')
270
281
  expect(p0.rhs[0].name).to eq('ELSE')
271
282
  expect(p0.rhs[1].name).to eq('stmt')
272
283
  expect(p0.name).to eq('return_children')
273
- expect(p0.constraints.size). to eq(1)
284
+ expect(p0.constraints.size).to eq(1)
274
285
  expect(p0.constraints[0]).to be_kind_of(Syntax::MatchClosest)
275
286
  expect(p0.constraints[0].idx_symbol).to eq(0) # ELSE is on position 0
276
287
  expect(p0.constraints[0].closest_symb).to eq('IF')
@@ -282,14 +293,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
282
293
  expect(p2.lhs.name).to eq('seq_ELSE_stmt_qmark')
283
294
  expect(p2.rhs).to be_empty
284
295
  expect(p2.name).to eq('_qmark_none')
285
-
286
- last_prod = instance.productions.last
287
- expect(last_prod.lhs.name).to eq('if_stmt')
288
- expect(last_prod.rhs.members[5].name).to eq('seq_ELSE_stmt_qmark')
289
296
  end
290
297
  end # context
291
298
  end # describe
292
299
  end # module
293
300
  end # module
294
301
 
295
- # End of file
302
+ # End of file