rley 0.8.00 → 0.8.05

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +47 -3
  3. data/CHANGELOG.md +32 -4
  4. data/examples/NLP/pico_en_demo.rb +2 -2
  5. data/examples/data_formats/JSON/README.md +34 -0
  6. data/examples/data_formats/JSON/sample01.json +3 -0
  7. data/examples/data_formats/JSON/sample01.svg +36 -0
  8. data/examples/data_formats/JSON/sample02.json +6 -0
  9. data/examples/data_formats/JSON/sample02.svg +128 -0
  10. data/examples/data_formats/JSON/sample03.json +88 -0
  11. data/examples/general/calc_iter1/README.md +26 -0
  12. data/examples/general/calc_iter2/README.md +55 -0
  13. data/examples/general/general_examples.md +37 -0
  14. data/examples/tokenizer/README.md +46 -0
  15. data/examples/tokenizer/loxxy_raw_scanner.rex +98 -0
  16. data/examples/tokenizer/loxxy_raw_scanner.rex.rb +256 -0
  17. data/examples/tokenizer/loxxy_tokenizer.rb +94 -0
  18. data/examples/tokenizer/run_tokenizer.rb +29 -0
  19. data/lib/rley/constants.rb +1 -1
  20. data/lib/rley/lexical/literal.rb +29 -0
  21. data/lib/rley/lexical/token.rb +7 -4
  22. data/lib/rley/notation/all_notation_nodes.rb +3 -1
  23. data/lib/rley/notation/ast_builder.rb +185 -191
  24. data/lib/rley/notation/ast_node.rb +5 -5
  25. data/lib/rley/notation/ast_visitor.rb +3 -1
  26. data/lib/rley/notation/grammar.rb +1 -1
  27. data/lib/rley/notation/grammar_builder.rb +87 -33
  28. data/lib/rley/notation/grouping_node.rb +1 -1
  29. data/lib/rley/notation/parser.rb +56 -56
  30. data/lib/rley/notation/sequence_node.rb +3 -3
  31. data/lib/rley/notation/symbol_node.rb +2 -2
  32. data/lib/rley/notation/tokenizer.rb +3 -15
  33. data/lib/rley/parse_rep/ast_base_builder.rb +35 -4
  34. data/lib/rley/parser/gfg_chart.rb +5 -4
  35. data/lib/rley/parser/gfg_earley_parser.rb +1 -1
  36. data/lib/rley/syntax/base_grammar_builder.rb +8 -2
  37. data/lib/rley/syntax/match_closest.rb +7 -7
  38. data/lib/rley.rb +1 -1
  39. data/spec/rley/lexical/literal_spec.rb +33 -0
  40. data/spec/rley/lexical/token_spec.rb +15 -4
  41. data/spec/rley/notation/grammar_builder_spec.rb +57 -50
  42. data/spec/rley/notation/parser_spec.rb +183 -184
  43. data/spec/rley/notation/tokenizer_spec.rb +98 -104
  44. data/spec/rley/parser/dangling_else_spec.rb +20 -20
  45. data/spec/rley/parser/gfg_chart_spec.rb +0 -1
  46. data/spec/rley/parser/gfg_earley_parser_spec.rb +166 -147
  47. data/spec/rley/parser/gfg_parsing_spec.rb +2 -2
  48. data/spec/rley/syntax/base_grammar_builder_spec.rb +7 -8
  49. data/spec/rley/syntax/grammar_spec.rb +6 -9
  50. data/spec/rley/syntax/match_closest_spec.rb +4 -4
  51. metadata +19 -9
  52. data/lib/rley/parser/parse_tracer.rb +0 -103
  53. data/lib/rley/syntax/literal.rb +0 -20
  54. data/lib/rley/syntax/verbatim_symbol.rb +0 -27
  55. data/spec/rley/syntax/literal_spec.rb +0 -31
  56. data/spec/rley/syntax/verbatim_symbol_spec.rb +0 -38
@@ -1,184 +1,183 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../../spec_helper' # Use the RSpec framework
4
-
5
- require_relative '../../../lib/rley/notation/ast_builder'
6
- # Load the class under test
7
- require_relative '../../../lib/rley/notation/parser'
8
-
9
- module Rley
10
- module Notation
11
- describe Parser do
12
- subject { Parser.new }
13
-
14
- # Utility method to walk towards deeply nested node
15
- # @param aNTNode [Rley::PTree::NonTerminalNode]
16
- # @param subnodePath[Array<Integer>] An Array of subnode indices
17
- def walk_subnodes(aNTNode, subnodePath)
18
- curr_node = aNTNode
19
- subnodePath.each do |index|
20
- curr_node = curr_node.subnodes[index]
21
- end
22
-
23
- curr_node
24
- end
25
-
26
- context 'Initialization:' do
27
- it 'should be initialized without argument' do
28
- expect { Parser.new }.not_to raise_error
29
- end
30
-
31
- it 'should have its parse engine initialized' do
32
- expect(subject.engine).to be_kind_of(Rley::Engine)
33
- end
34
- end # context
35
-
36
- context 'Parsing into CST:' do
37
- subject do
38
- instance = Parser.new
39
- instance.engine.configuration.repr_builder = Rley::ParseRep::CSTBuilder
40
-
41
- instance
42
- end
43
-
44
- it 'should parse single symbol names' do
45
- samples = %w[IF ifCondition statement]
46
-
47
- # One drawback od CSTs: they have a deeply nested structure
48
- samples.each do |source|
49
- ptree = subject.parse(source)
50
- expect(ptree.root).to be_kind_of(Rley::PTree::NonTerminalNode)
51
- expect(ptree.root.symbol.name).to eq('notation')
52
- expect(ptree.root.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
53
- expect(ptree.root.subnodes[0].symbol.name).to eq('rhs')
54
- expect(ptree.root.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
55
- member_seq = ptree.root.subnodes[0].subnodes[0]
56
- expect(member_seq.symbol.name).to eq('member_seq')
57
- expect(member_seq.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
58
- expect(member_seq.subnodes[0].symbol.name).to eq('member')
59
- expect(member_seq.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
60
- expect(member_seq.subnodes[0].subnodes[0].symbol.name).to eq('strait_member')
61
- strait_member = member_seq.subnodes[0].subnodes[0]
62
- expect(strait_member.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
63
- expect(strait_member.subnodes[0].symbol.name).to eq('base_member')
64
- expect(strait_member.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::TerminalNode)
65
- expect(strait_member.subnodes[0].subnodes[0].token.lexeme).to eq(source)
66
- end
67
- end
68
-
69
- end # context
70
-
71
- context 'Parsing into AST:' do
72
- subject do
73
- instance = Parser.new
74
- instance.engine.configuration.repr_builder = ASTBuilder
75
-
76
- instance
77
- end
78
-
79
- it 'should parse single symbol names' do
80
- samples = %w[IF ifCondition statement]
81
-
82
- samples.each do |source|
83
- ptree = subject.parse(source)
84
- expect(ptree.root).to be_kind_of(SymbolNode)
85
- expect(ptree.root.name).to eq(source)
86
- expect(ptree.root.repetition).to eq(:exactly_one)
87
- end
88
- end
89
-
90
- it 'should parse a sequence of symbols' do
91
- sequence = 'INT_LIT ELLIPSIS INT_LIT'
92
-
93
- ptree = subject.parse(sequence)
94
- expect(ptree.root).to be_kind_of(SequenceNode)
95
- expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
96
- expect(ptree.root.subnodes[0].name).to eq('INT_LIT')
97
- expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
98
- expect(ptree.root.subnodes[1].name).to eq('ELLIPSIS')
99
- expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
100
- expect(ptree.root.subnodes[2].name).to eq('INT_LIT')
101
- end
102
-
103
- it 'should parse an optional symbol' do
104
- optional = 'member_seq?'
105
-
106
- ptree = subject.parse(optional)
107
- expect(ptree.root).to be_kind_of(SymbolNode)
108
- expect(ptree.root.name).to eq('member_seq')
109
- expect(ptree.root.repetition).to eq(:zero_or_one)
110
- end
111
-
112
- it 'should parse a symbol with a + modifier' do
113
- one_or_more = 'member+'
114
-
115
- ptree = subject.parse(one_or_more)
116
- expect(ptree.root).to be_kind_of(SymbolNode)
117
- expect(ptree.root.name).to eq('member')
118
- expect(ptree.root.repetition).to eq(:one_or_more)
119
- end
120
-
121
- it 'should parse a symbol with a * modifier' do
122
- zero_or_more = 'declaration* EOF'
123
-
124
- ptree = subject.parse(zero_or_more)
125
- expect(ptree.root).to be_kind_of(SequenceNode)
126
- expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
127
- expect(ptree.root.subnodes[0].name).to eq('declaration')
128
- expect(ptree.root.subnodes[0].repetition).to eq(:zero_or_more)
129
- expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
130
- expect(ptree.root.subnodes[1].name).to eq('EOF')
131
- expect(ptree.root.subnodes[1].repetition).to eq(:exactly_one)
132
- end
133
-
134
- it 'should parse a grouping with a modifier' do
135
- input = "IF ifCondition statement (ELSE statement)?"
136
-
137
- ptree = subject.parse(input)
138
- expect(ptree.root).to be_kind_of(SequenceNode)
139
- expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
140
- expect(ptree.root.subnodes[0].name).to eq('IF')
141
- expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
142
- expect(ptree.root.subnodes[1].name).to eq('ifCondition')
143
- expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
144
- expect(ptree.root.subnodes[2].name).to eq('statement')
145
- expect(ptree.root.subnodes[3]).to be_kind_of(SequenceNode)
146
- expect(ptree.root.subnodes[3].repetition).to eq(:zero_or_one)
147
- expect(ptree.root.subnodes[3].subnodes[0]).to be_kind_of(SymbolNode)
148
- expect(ptree.root.subnodes[3].subnodes[0].name).to eq('ELSE')
149
- expect(ptree.root.subnodes[3].subnodes[1]).to be_kind_of(SymbolNode)
150
- expect(ptree.root.subnodes[3].subnodes[1].name).to eq('statement')
151
- end
152
-
153
- it 'should parse an annotated symbol' do
154
- optional = 'member_seq{repeat: 0..1}'
155
-
156
- ptree = subject.parse(optional)
157
- expect(ptree.root).to be_kind_of(SymbolNode)
158
- expect(ptree.root.name).to eq('member_seq')
159
- expect(ptree.root.repetition).to eq(:zero_or_one)
160
- end
161
-
162
- it 'should parse a grouping with embedded annotation' do
163
- if_stmt = "IF ifCondition statement ( ELSE { match_closest: 'IF' } statement )?"
164
-
165
- ptree = subject.parse(if_stmt)
166
- expect(ptree.root).to be_kind_of(SequenceNode)
167
- expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
168
- expect(ptree.root.subnodes[0].name).to eq('IF')
169
- expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
170
- expect(ptree.root.subnodes[1].name).to eq('ifCondition')
171
- expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
172
- expect(ptree.root.subnodes[2].name).to eq('statement')
173
- optional = ptree.root.subnodes[3]
174
- expect(optional).to be_kind_of(SequenceNode)
175
- expect(optional.repetition).to eq(:zero_or_one)
176
- expect(optional.subnodes[0]).to be_kind_of(SymbolNode)
177
- expect(optional.subnodes[0].name).to eq('ELSE')
178
- expect(optional.subnodes[0].annotation).to eq({'match_closest' => 'IF'})
179
- expect(optional.subnodes[1].name).to eq('statement')
180
- end
181
- end # context
182
- end # describe
183
- end # module
184
- end # module
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../../spec_helper' # Use the RSpec framework
4
+
5
+ require_relative '../../../lib/rley/notation/ast_builder'
6
+ # Load the class under test
7
+ require_relative '../../../lib/rley/notation/parser'
8
+
9
+ module Rley
10
+ module Notation
11
+ describe Parser do
12
+ subject { Parser.new }
13
+
14
+ # Utility method to walk towards deeply nested node
15
+ # @param aNTNode [Rley::PTree::NonTerminalNode]
16
+ # @param subnodePath[Array<Integer>] An Array of subnode indices
17
+ def walk_subnodes(aNTNode, subnodePath)
18
+ curr_node = aNTNode
19
+ subnodePath.each do |index|
20
+ curr_node = curr_node.subnodes[index]
21
+ end
22
+
23
+ curr_node
24
+ end
25
+
26
+ context 'Initialization:' do
27
+ it 'should be initialized without argument' do
28
+ expect { Parser.new }.not_to raise_error
29
+ end
30
+
31
+ it 'should have its parse engine initialized' do
32
+ expect(subject.engine).to be_kind_of(Rley::Engine)
33
+ end
34
+ end # context
35
+
36
+ context 'Parsing into CST:' do
37
+ subject do
38
+ instance = Parser.new
39
+ instance.engine.configuration.repr_builder = Rley::ParseRep::CSTBuilder
40
+
41
+ instance
42
+ end
43
+
44
+ it 'should parse single symbol names' do
45
+ samples = %w[IF ifCondition statement]
46
+
47
+ # One drawback od CSTs: they have a deeply nested structure
48
+ samples.each do |source|
49
+ ptree = subject.parse(source)
50
+ expect(ptree.root).to be_kind_of(Rley::PTree::NonTerminalNode)
51
+ expect(ptree.root.symbol.name).to eq('notation')
52
+ expect(ptree.root.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
53
+ expect(ptree.root.subnodes[0].symbol.name).to eq('rhs')
54
+ expect(ptree.root.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
55
+ member_seq = ptree.root.subnodes[0].subnodes[0]
56
+ expect(member_seq.symbol.name).to eq('member_seq')
57
+ expect(member_seq.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
58
+ expect(member_seq.subnodes[0].symbol.name).to eq('member')
59
+ expect(member_seq.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
60
+ expect(member_seq.subnodes[0].subnodes[0].symbol.name).to eq('strait_member')
61
+ strait_member = member_seq.subnodes[0].subnodes[0]
62
+ expect(strait_member.subnodes[0]).to be_kind_of(Rley::PTree::NonTerminalNode)
63
+ expect(strait_member.subnodes[0].symbol.name).to eq('base_member')
64
+ expect(strait_member.subnodes[0].subnodes[0]).to be_kind_of(Rley::PTree::TerminalNode)
65
+ expect(strait_member.subnodes[0].subnodes[0].token.lexeme).to eq(source)
66
+ end
67
+ end
68
+ end # context
69
+
70
+ context 'Parsing into AST:' do
71
+ subject do
72
+ instance = Parser.new
73
+ instance.engine.configuration.repr_builder = ASTBuilder
74
+
75
+ instance
76
+ end
77
+
78
+ it 'should parse single symbol names' do
79
+ samples = %w[IF ifCondition statement]
80
+
81
+ samples.each do |source|
82
+ ptree = subject.parse(source)
83
+ expect(ptree.root).to be_kind_of(SymbolNode)
84
+ expect(ptree.root.name).to eq(source)
85
+ expect(ptree.root.repetition).to eq(:exactly_one)
86
+ end
87
+ end
88
+
89
+ it 'should parse a sequence of symbols' do
90
+ sequence = 'INT_LIT ELLIPSIS INT_LIT'
91
+
92
+ ptree = subject.parse(sequence)
93
+ expect(ptree.root).to be_kind_of(SequenceNode)
94
+ expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
95
+ expect(ptree.root.subnodes[0].name).to eq('INT_LIT')
96
+ expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
97
+ expect(ptree.root.subnodes[1].name).to eq('ELLIPSIS')
98
+ expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
99
+ expect(ptree.root.subnodes[2].name).to eq('INT_LIT')
100
+ end
101
+
102
+ it 'should parse an optional symbol' do
103
+ optional = 'member_seq?'
104
+
105
+ ptree = subject.parse(optional)
106
+ expect(ptree.root).to be_kind_of(SymbolNode)
107
+ expect(ptree.root.name).to eq('member_seq')
108
+ expect(ptree.root.repetition).to eq(:zero_or_one)
109
+ end
110
+
111
+ it 'should parse a symbol with a + modifier' do
112
+ one_or_more = 'member+'
113
+
114
+ ptree = subject.parse(one_or_more)
115
+ expect(ptree.root).to be_kind_of(SymbolNode)
116
+ expect(ptree.root.name).to eq('member')
117
+ expect(ptree.root.repetition).to eq(:one_or_more)
118
+ end
119
+
120
+ it 'should parse a symbol with a * modifier' do
121
+ zero_or_more = 'declaration* EOF'
122
+
123
+ ptree = subject.parse(zero_or_more)
124
+ expect(ptree.root).to be_kind_of(SequenceNode)
125
+ expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
126
+ expect(ptree.root.subnodes[0].name).to eq('declaration')
127
+ expect(ptree.root.subnodes[0].repetition).to eq(:zero_or_more)
128
+ expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
129
+ expect(ptree.root.subnodes[1].name).to eq('EOF')
130
+ expect(ptree.root.subnodes[1].repetition).to eq(:exactly_one)
131
+ end
132
+
133
+ it 'should parse a grouping with a modifier' do
134
+ input = 'IF ifCondition statement (ELSE statement)?'
135
+
136
+ ptree = subject.parse(input)
137
+ expect(ptree.root).to be_kind_of(SequenceNode)
138
+ expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
139
+ expect(ptree.root.subnodes[0].name).to eq('IF')
140
+ expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
141
+ expect(ptree.root.subnodes[1].name).to eq('ifCondition')
142
+ expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
143
+ expect(ptree.root.subnodes[2].name).to eq('statement')
144
+ expect(ptree.root.subnodes[3]).to be_kind_of(SequenceNode)
145
+ expect(ptree.root.subnodes[3].repetition).to eq(:zero_or_one)
146
+ expect(ptree.root.subnodes[3].subnodes[0]).to be_kind_of(SymbolNode)
147
+ expect(ptree.root.subnodes[3].subnodes[0].name).to eq('ELSE')
148
+ expect(ptree.root.subnodes[3].subnodes[1]).to be_kind_of(SymbolNode)
149
+ expect(ptree.root.subnodes[3].subnodes[1].name).to eq('statement')
150
+ end
151
+
152
+ it 'should parse an annotated symbol' do
153
+ optional = 'member_seq{repeat: 0..1}'
154
+
155
+ ptree = subject.parse(optional)
156
+ expect(ptree.root).to be_kind_of(SymbolNode)
157
+ expect(ptree.root.name).to eq('member_seq')
158
+ expect(ptree.root.repetition).to eq(:zero_or_one)
159
+ end
160
+
161
+ it 'should parse a grouping with embedded annotation' do
162
+ if_stmt = "IF ifCondition statement ( ELSE { match_closest: 'IF' } statement )?"
163
+
164
+ ptree = subject.parse(if_stmt)
165
+ expect(ptree.root).to be_kind_of(SequenceNode)
166
+ expect(ptree.root.subnodes[0]).to be_kind_of(SymbolNode)
167
+ expect(ptree.root.subnodes[0].name).to eq('IF')
168
+ expect(ptree.root.subnodes[1]).to be_kind_of(SymbolNode)
169
+ expect(ptree.root.subnodes[1].name).to eq('ifCondition')
170
+ expect(ptree.root.subnodes[2]).to be_kind_of(SymbolNode)
171
+ expect(ptree.root.subnodes[2].name).to eq('statement')
172
+ optional = ptree.root.subnodes[3]
173
+ expect(optional).to be_kind_of(SequenceNode)
174
+ expect(optional.repetition).to eq(:zero_or_one)
175
+ expect(optional.subnodes[0]).to be_kind_of(SymbolNode)
176
+ expect(optional.subnodes[0].name).to eq('ELSE')
177
+ expect(optional.subnodes[0].annotation).to eq({ 'match_closest' => 'IF' })
178
+ expect(optional.subnodes[1].name).to eq('statement')
179
+ end
180
+ end # context
181
+ end # describe
182
+ end # module
183
+ end # module