rley 0.7.08 → 0.8.03

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +29 -5
  3. data/CHANGELOG.md +28 -4
  4. data/README.md +4 -5
  5. data/examples/NLP/nano_eng/nano_en_demo.rb +7 -11
  6. data/examples/NLP/nano_eng/nano_grammar.rb +18 -18
  7. data/examples/data_formats/JSON/json_ast_builder.rb +9 -18
  8. data/examples/data_formats/JSON/json_demo.rb +1 -2
  9. data/examples/data_formats/JSON/json_grammar.rb +11 -11
  10. data/examples/general/calc_iter1/calc_grammar.rb +5 -4
  11. data/examples/general/calc_iter2/calc_grammar.rb +9 -9
  12. data/examples/general/left.rb +1 -1
  13. data/examples/general/right.rb +1 -1
  14. data/lib/rley/base/dotted_item.rb +5 -0
  15. data/lib/rley/base/grm_items_builder.rb +6 -0
  16. data/lib/rley/constants.rb +1 -1
  17. data/lib/rley/engine.rb +2 -2
  18. data/lib/rley/interface.rb +16 -0
  19. data/lib/rley/notation/all_notation_nodes.rb +4 -0
  20. data/lib/rley/notation/ast_builder.rb +185 -0
  21. data/lib/rley/notation/ast_node.rb +44 -0
  22. data/lib/rley/notation/ast_visitor.rb +115 -0
  23. data/lib/rley/notation/grammar.rb +49 -0
  24. data/lib/rley/notation/grammar_builder.rb +505 -0
  25. data/lib/rley/notation/grouping_node.rb +23 -0
  26. data/lib/rley/notation/parser.rb +56 -0
  27. data/lib/rley/notation/sequence_node.rb +35 -0
  28. data/lib/rley/notation/symbol_node.rb +29 -0
  29. data/lib/rley/notation/tokenizer.rb +180 -0
  30. data/lib/rley/parse_rep/ast_base_builder.rb +44 -0
  31. data/lib/rley/parser/gfg_chart.rb +101 -6
  32. data/lib/rley/parser/gfg_earley_parser.rb +1 -1
  33. data/lib/rley/parser/gfg_parsing.rb +5 -3
  34. data/lib/rley/parser/parse_entry_set.rb +1 -1
  35. data/lib/rley/syntax/{grammar_builder.rb → base_grammar_builder.rb} +53 -15
  36. data/lib/rley/syntax/grm_symbol.rb +1 -1
  37. data/lib/rley/syntax/match_closest.rb +43 -0
  38. data/lib/rley/syntax/production.rb +6 -0
  39. data/lib/rley.rb +1 -1
  40. data/spec/rley/engine_spec.rb +6 -6
  41. data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -2
  42. data/spec/rley/notation/grammar_builder_spec.rb +302 -0
  43. data/spec/rley/notation/parser_spec.rb +183 -0
  44. data/spec/rley/notation/tokenizer_spec.rb +364 -0
  45. data/spec/rley/parse_rep/ast_builder_spec.rb +0 -1
  46. data/spec/rley/parse_rep/groucho_spec.rb +1 -1
  47. data/spec/rley/parse_rep/parse_forest_builder_spec.rb +1 -1
  48. data/spec/rley/parse_rep/parse_forest_factory_spec.rb +2 -2
  49. data/spec/rley/parse_rep/parse_tree_factory_spec.rb +1 -1
  50. data/spec/rley/parser/dangling_else_spec.rb +447 -0
  51. data/spec/rley/parser/gfg_earley_parser_spec.rb +118 -10
  52. data/spec/rley/parser/gfg_parsing_spec.rb +2 -1
  53. data/spec/rley/parser/parse_walker_factory_spec.rb +2 -2
  54. data/spec/rley/support/ambiguous_grammar_helper.rb +2 -2
  55. data/spec/rley/support/grammar_abc_helper.rb +2 -2
  56. data/spec/rley/support/grammar_ambig01_helper.rb +2 -2
  57. data/spec/rley/support/grammar_arr_int_helper.rb +2 -2
  58. data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
  59. data/spec/rley/support/grammar_int_seq_helper.rb +51 -0
  60. data/spec/rley/support/grammar_l0_helper.rb +2 -2
  61. data/spec/rley/support/grammar_pb_helper.rb +2 -2
  62. data/spec/rley/support/grammar_sppf_helper.rb +2 -2
  63. data/spec/rley/syntax/{grammar_builder_spec.rb → base_grammar_builder_spec.rb} +29 -11
  64. data/spec/rley/syntax/match_closest_spec.rb +46 -0
  65. data/spec/rley/syntax/production_spec.rb +4 -0
  66. metadata +29 -14
  67. data/lib/rley/parser/parse_state.rb +0 -78
  68. data/lib/rley/parser/parse_state_tracker.rb +0 -59
  69. data/lib/rley/parser/state_set.rb +0 -100
  70. data/spec/rley/parser/parse_state_spec.rb +0 -125
  71. data/spec/rley/parser/parse_tracer_spec.rb +0 -200
  72. data/spec/rley/parser/state_set_spec.rb +0 -130
@@ -1,200 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../../spec_helper'
4
- require 'stringio'
5
-
6
- require_relative '../../../lib/rley/syntax/terminal'
7
- require_relative '../../../lib/rley/syntax/non_terminal'
8
- require_relative '../../../lib/rley/syntax/production'
9
- require_relative '../../../lib/rley/base/dotted_item'
10
- require_relative '../../../lib/rley/parser/parse_state'
11
- require_relative '../../../lib/rley/lexical/token'
12
-
13
- # Load the class under test
14
- require_relative '../../../lib/rley/parser/parse_tracer'
15
-
16
- module Rley # Open this namespace to avoid module qualifier prefixes
17
- module Parser # Open this namespace to avoid module qualifier prefixes
18
- describe ParseTracer do
19
- let(:output) { StringIO.new(+'', 'w') }
20
- let(:tpos) { Lexical::Position.new(3, 4) }
21
-
22
- let(:token_seq) do
23
- literals = %w[I saw John with a dog]
24
- literals.map do |lexeme|
25
- Lexical::Token.new(lexeme, double('fake-terminal'), tpos)
26
- end
27
- end
28
-
29
- subject { ParseTracer.new(1, output, token_seq) }
30
-
31
- context 'Creation & initialization:' do
32
- it 'should accept trace level 0' do
33
- expect { ParseTracer.new(0, output, token_seq) }.not_to raise_error
34
- expect(output.string).to eq('')
35
- end
36
-
37
-
38
- it 'should accept trace level 1' do
39
- expect { ParseTracer.new(1, output, token_seq) }.not_to raise_error
40
- expectations = <<-SNIPPET
41
- ['I', 'saw', 'John', 'with', 'a', 'dog']
42
- |. I . saw . John . with . a . dog .|
43
- SNIPPET
44
- expect(output.string).to eq(expectations)
45
- end
46
-
47
- it 'should accept trace level 2' do
48
- expect { ParseTracer.new(2, output, token_seq) }.not_to raise_error
49
- expectations = <<-SNIPPET
50
- ['I', 'saw', 'John', 'with', 'a', 'dog']
51
- |. I . saw . John . with . a . dog .|
52
- SNIPPET
53
- expect(output.string).to eq(expectations)
54
- end
55
-
56
- it 'should know the trace level' do
57
- expect(subject.level).to eq(1)
58
- end
59
-
60
- it 'should know the output stream' do
61
- expect(subject.ostream).to eq(output)
62
- end
63
- end # context
64
-
65
- context 'Provided services:' do
66
- let(:t_a) { Syntax::Terminal.new('A') }
67
- let(:t_b) { Syntax::Terminal.new('B') }
68
- let(:t_c) { Syntax::Terminal.new('C') }
69
- let(:nt_sentence) { Syntax::NonTerminal.new('sentence') }
70
-
71
- let(:sample_prod) do
72
- Syntax::Production.new(nt_sentence, [t_a, t_b, t_c])
73
- end
74
-
75
- let(:origin_val) { 3 }
76
- let(:dotted_rule) { Base::DottedItem.new(sample_prod, 2) }
77
- let(:complete_rule) { Base::DottedItem.new(sample_prod, 3) }
78
- let(:sample_parse_state) { ParseState.new(dotted_rule, origin_val) }
79
-
80
- # Factory method.
81
- def parse_state(origin, aDottedRule)
82
- ParseState.new(aDottedRule, origin)
83
- end
84
-
85
- it 'should render a scanning step' do
86
- # Case: token at the beginning
87
- subject.ostream.string = +''
88
- subject.trace_scanning(1, parse_state(0, dotted_rule))
89
- expectations = <<-SNIPPET
90
- |[------] . . . . .| [0:1] sentence => A B . C
91
- SNIPPET
92
- expect(output.string).to eq(expectations)
93
-
94
- # Case: token in the middle
95
- subject.ostream.string = +''
96
- subject.trace_scanning(4, sample_parse_state)
97
- expectations = <<-SNIPPET
98
- |. . . [------] . .| [3:4] sentence => A B . C
99
- SNIPPET
100
- expect(output.string).to eq(expectations)
101
-
102
- # Case: token at the end
103
- subject.ostream.string = +''
104
- subject.trace_scanning(6, parse_state(5, dotted_rule))
105
- expectations = <<-SNIPPET
106
- |. . . . . [------]| [5:6] sentence => A B . C
107
- SNIPPET
108
- expect(output.string).to eq(expectations)
109
- end
110
-
111
-
112
- it 'should render a prediction step' do
113
- # Case: initial stateset
114
- subject.ostream.string = +''
115
- subject.trace_prediction(0, parse_state(0, dotted_rule))
116
- expectations = <<-SNIPPET
117
- |> . . . . . .| [0:0] sentence => A B . C
118
- SNIPPET
119
- expect(output.string).to eq(expectations)
120
-
121
- # Case: stateset in the middle
122
- subject.ostream.string = +''
123
- subject.trace_prediction(3, sample_parse_state)
124
- expectations = <<-SNIPPET
125
- |. . . > . . .| [3:3] sentence => A B . C
126
- SNIPPET
127
- expect(output.string).to eq(expectations)
128
-
129
- # Case: final stateset
130
- subject.ostream.string = +''
131
- subject.trace_prediction(6, parse_state(6, dotted_rule))
132
- expectations = <<-SNIPPET
133
- |. . . . . . >| [6:6] sentence => A B . C
134
- SNIPPET
135
- expect(output.string).to eq(expectations)
136
- end
137
-
138
- it 'should render a completion step' do
139
- # Case: full parse completed
140
- subject.ostream.string = +''
141
- subject.trace_completion(6, parse_state(0, complete_rule))
142
- expectations = <<-SNIPPET
143
- |[=========================================]| [0:6] sentence => A B C .
144
- SNIPPET
145
- expect(output.string).to eq(expectations)
146
-
147
- # Case: step at the start (complete)
148
- subject.ostream.string = +''
149
- subject.trace_completion(1, parse_state(0, complete_rule))
150
- expectations = <<-SNIPPET
151
- |[------] . . . . .| [0:1] sentence => A B C .
152
- SNIPPET
153
- expect(output.string).to eq(expectations)
154
-
155
- # Case: step at the start (not complete)
156
- subject.ostream.string = +''
157
- subject.trace_completion(1, parse_state(0, dotted_rule))
158
- expectations = <<-SNIPPET
159
- |[------> . . . . .| [0:1] sentence => A B . C
160
- SNIPPET
161
- expect(output.string).to eq(expectations)
162
-
163
- # Case: step at the middle (complete)
164
- subject.ostream.string = +''
165
- subject.trace_completion(4, parse_state(2, complete_rule))
166
- expectations = <<-SNIPPET
167
- |. . [-------------] . .| [2:4] sentence => A B C .
168
- SNIPPET
169
- expect(output.string).to eq(expectations)
170
-
171
- # Case: step at the middle (not complete)
172
- subject.ostream.string = +''
173
- subject.trace_completion(4, parse_state(2, dotted_rule))
174
- expectations = <<-SNIPPET
175
- |. . [-------------> . .| [2:4] sentence => A B . C
176
- SNIPPET
177
- expect(output.string).to eq(expectations)
178
-
179
- # Case: step at the end (complete)
180
- subject.ostream.string = +''
181
- subject.trace_completion(6, parse_state(3, complete_rule))
182
- expectations = <<-SNIPPET
183
- |. . . [--------------------]| [3:6] sentence => A B C .
184
- SNIPPET
185
- expect(output.string).to eq(expectations)
186
-
187
- # Case: step at the end (not complete)
188
- subject.ostream.string = +''
189
- subject.trace_completion(6, parse_state(3, dotted_rule))
190
- expectations = <<-SNIPPET
191
- |. . . [-------------------->| [3:6] sentence => A B . C
192
- SNIPPET
193
- expect(output.string).to eq(expectations)
194
- end
195
- end # context
196
- end # describe
197
- end # module
198
- end # module
199
-
200
- # End of file
@@ -1,130 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../../spec_helper'
4
-
5
- require_relative '../../../lib/rley/parser/parse_state'
6
-
7
- # Load the class under test
8
- require_relative '../../../lib/rley/parser/state_set'
9
-
10
- module Rley # Open this namespace to avoid module qualifier prefixes
11
- module Parser # Open this namespace to avoid module qualifier prefixes
12
- describe StateSet do
13
- let(:dotted_rule1) { double('fake_dotted_rule1') }
14
- let(:state1) { ParseState.new(dotted_rule1, 2) }
15
- let(:dotted_rule2) { double('fake_dotted_rule2') }
16
- let(:state2) { ParseState.new(dotted_rule2, 5) }
17
-
18
- context 'Initialization:' do
19
- it 'should be created without argument' do
20
- expect { StateSet.new }.not_to raise_error
21
- end
22
-
23
- it 'should be empty at creation' do
24
- expect(subject.states).to be_empty
25
- end
26
- end # context
27
-
28
- context 'Provided services:' do
29
- it 'should push a state' do
30
- expect(subject.states).to be_empty
31
- expect { subject.push_state(state1) }.not_to raise_error
32
- expect(subject).not_to be_empty
33
- subject.push_state(state2)
34
- expect(subject.states).to eq([state1, state2])
35
- end
36
-
37
- it 'should ignore a second push of a state' do
38
- expect(subject.states).to be_empty
39
- subject.push_state(state1)
40
- subject.push_state(state2)
41
- expect(subject.states).to eq([state1, state2])
42
-
43
- # One tries to push an already pushed state
44
- expect(subject.push_state(state1)).to be_falsy
45
-
46
- # ...It is not added
47
- expect(subject.states).to eq([state1, state2])
48
- end
49
-
50
- it 'should list the states expecting a given terminal' do
51
- # Case of no state
52
- expect(subject.states_expecting(:a)).to be_empty
53
-
54
- # Adding states
55
- subject.push_state(state1)
56
- subject.push_state(state2)
57
- allow(dotted_rule1).to receive(:next_symbol).and_return(:b)
58
- allow(dotted_rule2).to receive(:next_symbol).and_return(:a)
59
- expect(subject.states_expecting(:a)).to eq([state2])
60
- expect(subject.states_expecting(:b)).to eq([state1])
61
- end
62
-
63
- it 'should list the states related to a production' do
64
- a_prod = double('fake-production')
65
-
66
- # Case of no state
67
- expect(subject.states_for(a_prod)).to be_empty
68
-
69
- # Adding states
70
- subject.push_state(state1)
71
- subject.push_state(state2)
72
- expect(dotted_rule1).to receive(:production).and_return(:dummy)
73
- expect(dotted_rule2).to receive(:production).and_return(a_prod)
74
- expect(subject.states_for(a_prod)).to eq([state2])
75
- end
76
-
77
- it 'should list the states that rewrite a given non-terminal' do
78
- non_term = double('fake-non-terminal')
79
- prod1 = double('fake-production1')
80
- prod2 = double('fake-production2')
81
-
82
- # Adding states
83
- subject.push_state(state1)
84
- subject.push_state(state2)
85
- expect(dotted_rule1).to receive(:production).and_return(prod1)
86
- expect(prod1).to receive(:lhs).and_return(:dummy)
87
- expect(dotted_rule2).to receive(:production).and_return(prod2)
88
- expect(dotted_rule2).to receive(:reduce_item?).and_return(true)
89
- expect(prod2).to receive(:lhs).and_return(non_term)
90
- expect(subject.states_rewriting(non_term)).to eq([state2])
91
- end
92
-
93
- it 'should list of ambiguous states' do
94
- prod1 = double('fake-production1')
95
- prod2 = double('fake-production2')
96
- expect(subject.ambiguities.size).to eq(0)
97
-
98
- # Adding states
99
- subject.push_state(state1)
100
- allow(dotted_rule1).to receive(:production).and_return(prod1)
101
- allow(dotted_rule1).to receive(:reduce_item?).and_return(true)
102
- allow(dotted_rule1).to receive(:lhs).and_return(:something)
103
- expect(subject.ambiguities.size).to eq(0)
104
- allow(dotted_rule2).to receive(:production).and_return(prod2)
105
- allow(dotted_rule2).to receive(:reduce_item?).and_return(true)
106
- allow(dotted_rule2).to receive(:lhs).and_return(:something_else)
107
- subject.push_state(state2)
108
- expect(subject.ambiguities.size).to eq(0)
109
- dotted_rule3 = double('fake_dotted_rule3')
110
- allow(dotted_rule3).to receive(:production).and_return(prod2)
111
- allow(dotted_rule3).to receive(:reduce_item?).and_return(true)
112
- allow(dotted_rule3).to receive(:lhs).and_return(:something_else)
113
- state3 = ParseState.new(dotted_rule3, 5)
114
- subject.push_state(state3)
115
- expect(subject.ambiguities[0]).to eq([state2, state3])
116
- end
117
-
118
- it 'should complain when impossible predecessor of parse state' do
119
- subject.push_state(state1)
120
- subject.push_state(state2)
121
- expect(dotted_rule1).to receive(:prev_position).and_return(nil)
122
- err = StandardError
123
- expect { subject.predecessor_state(state1) }.to raise_error(err)
124
- end
125
- end # context
126
- end # describe
127
- end # module
128
- end # module
129
-
130
- # End of file