rley 0.7.08 → 0.8.00

Sign up to get free protection for your applications and to get access to all the features.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -0
  3. data/README.md +4 -5
  4. data/examples/NLP/nano_eng/nano_en_demo.rb +7 -11
  5. data/examples/NLP/nano_eng/nano_grammar.rb +18 -18
  6. data/examples/NLP/pico_en_demo.rb +2 -2
  7. data/examples/data_formats/JSON/json_ast_builder.rb +9 -18
  8. data/examples/data_formats/JSON/json_demo.rb +1 -2
  9. data/examples/data_formats/JSON/json_grammar.rb +11 -11
  10. data/examples/general/calc_iter1/calc_grammar.rb +5 -4
  11. data/examples/general/calc_iter2/calc_grammar.rb +9 -9
  12. data/examples/general/left.rb +1 -1
  13. data/examples/general/right.rb +1 -1
  14. data/lib/rley.rb +1 -1
  15. data/lib/rley/base/dotted_item.rb +5 -0
  16. data/lib/rley/base/grm_items_builder.rb +6 -0
  17. data/lib/rley/constants.rb +1 -1
  18. data/lib/rley/engine.rb +2 -2
  19. data/lib/rley/interface.rb +16 -0
  20. data/lib/rley/notation/all_notation_nodes.rb +2 -0
  21. data/lib/rley/notation/ast_builder.rb +191 -0
  22. data/lib/rley/notation/ast_node.rb +44 -0
  23. data/lib/rley/notation/ast_visitor.rb +113 -0
  24. data/lib/rley/notation/grammar.rb +49 -0
  25. data/lib/rley/notation/grammar_builder.rb +451 -0
  26. data/lib/rley/notation/grouping_node.rb +23 -0
  27. data/lib/rley/notation/parser.rb +56 -0
  28. data/lib/rley/notation/sequence_node.rb +35 -0
  29. data/lib/rley/notation/symbol_node.rb +29 -0
  30. data/lib/rley/notation/tokenizer.rb +192 -0
  31. data/lib/rley/parse_rep/ast_base_builder.rb +13 -0
  32. data/lib/rley/parser/gfg_chart.rb +100 -6
  33. data/lib/rley/parser/gfg_parsing.rb +5 -3
  34. data/lib/rley/parser/parse_entry_set.rb +1 -1
  35. data/lib/rley/syntax/{grammar_builder.rb → base_grammar_builder.rb} +45 -15
  36. data/lib/rley/syntax/grm_symbol.rb +1 -1
  37. data/lib/rley/syntax/match_closest.rb +43 -0
  38. data/lib/rley/syntax/production.rb +6 -0
  39. data/spec/rley/engine_spec.rb +6 -6
  40. data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -2
  41. data/spec/rley/notation/grammar_builder_spec.rb +295 -0
  42. data/spec/rley/notation/parser_spec.rb +184 -0
  43. data/spec/rley/notation/tokenizer_spec.rb +370 -0
  44. data/spec/rley/parse_rep/ast_builder_spec.rb +0 -1
  45. data/spec/rley/parse_rep/groucho_spec.rb +1 -1
  46. data/spec/rley/parse_rep/parse_forest_builder_spec.rb +1 -1
  47. data/spec/rley/parse_rep/parse_forest_factory_spec.rb +2 -2
  48. data/spec/rley/parse_rep/parse_tree_factory_spec.rb +1 -1
  49. data/spec/rley/parser/dangling_else_spec.rb +445 -0
  50. data/spec/rley/parser/gfg_earley_parser_spec.rb +95 -9
  51. data/spec/rley/parser/gfg_parsing_spec.rb +1 -1
  52. data/spec/rley/parser/parse_walker_factory_spec.rb +2 -2
  53. data/spec/rley/support/ambiguous_grammar_helper.rb +2 -2
  54. data/spec/rley/support/grammar_abc_helper.rb +2 -2
  55. data/spec/rley/support/grammar_ambig01_helper.rb +2 -2
  56. data/spec/rley/support/grammar_arr_int_helper.rb +2 -2
  57. data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
  58. data/spec/rley/support/grammar_int_seq_helper.rb +51 -0
  59. data/spec/rley/support/grammar_l0_helper.rb +2 -2
  60. data/spec/rley/support/grammar_pb_helper.rb +2 -2
  61. data/spec/rley/support/grammar_sppf_helper.rb +2 -2
  62. data/spec/rley/syntax/{grammar_builder_spec.rb → base_grammar_builder_spec.rb} +30 -11
  63. data/spec/rley/syntax/match_closest_spec.rb +46 -0
  64. data/spec/rley/syntax/production_spec.rb +4 -0
  65. metadata +29 -14
  66. data/lib/rley/parser/parse_state.rb +0 -78
  67. data/lib/rley/parser/parse_state_tracker.rb +0 -59
  68. data/lib/rley/parser/state_set.rb +0 -100
  69. data/spec/rley/parser/parse_state_spec.rb +0 -125
  70. data/spec/rley/parser/parse_tracer_spec.rb +0 -200
  71. data/spec/rley/parser/state_set_spec.rb +0 -130
@@ -1,200 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../../spec_helper'
4
- require 'stringio'
5
-
6
- require_relative '../../../lib/rley/syntax/terminal'
7
- require_relative '../../../lib/rley/syntax/non_terminal'
8
- require_relative '../../../lib/rley/syntax/production'
9
- require_relative '../../../lib/rley/base/dotted_item'
10
- require_relative '../../../lib/rley/parser/parse_state'
11
- require_relative '../../../lib/rley/lexical/token'
12
-
13
- # Load the class under test
14
- require_relative '../../../lib/rley/parser/parse_tracer'
15
-
16
- module Rley # Open this namespace to avoid module qualifier prefixes
17
- module Parser # Open this namespace to avoid module qualifier prefixes
18
- describe ParseTracer do
19
- let(:output) { StringIO.new(+'', 'w') }
20
- let(:tpos) { Lexical::Position.new(3, 4) }
21
-
22
- let(:token_seq) do
23
- literals = %w[I saw John with a dog]
24
- literals.map do |lexeme|
25
- Lexical::Token.new(lexeme, double('fake-terminal'), tpos)
26
- end
27
- end
28
-
29
- subject { ParseTracer.new(1, output, token_seq) }
30
-
31
- context 'Creation & initialization:' do
32
- it 'should accept trace level 0' do
33
- expect { ParseTracer.new(0, output, token_seq) }.not_to raise_error
34
- expect(output.string).to eq('')
35
- end
36
-
37
-
38
- it 'should accept trace level 1' do
39
- expect { ParseTracer.new(1, output, token_seq) }.not_to raise_error
40
- expectations = <<-SNIPPET
41
- ['I', 'saw', 'John', 'with', 'a', 'dog']
42
- |. I . saw . John . with . a . dog .|
43
- SNIPPET
44
- expect(output.string).to eq(expectations)
45
- end
46
-
47
- it 'should accept trace level 2' do
48
- expect { ParseTracer.new(2, output, token_seq) }.not_to raise_error
49
- expectations = <<-SNIPPET
50
- ['I', 'saw', 'John', 'with', 'a', 'dog']
51
- |. I . saw . John . with . a . dog .|
52
- SNIPPET
53
- expect(output.string).to eq(expectations)
54
- end
55
-
56
- it 'should know the trace level' do
57
- expect(subject.level).to eq(1)
58
- end
59
-
60
- it 'should know the output stream' do
61
- expect(subject.ostream).to eq(output)
62
- end
63
- end # context
64
-
65
- context 'Provided services:' do
66
- let(:t_a) { Syntax::Terminal.new('A') }
67
- let(:t_b) { Syntax::Terminal.new('B') }
68
- let(:t_c) { Syntax::Terminal.new('C') }
69
- let(:nt_sentence) { Syntax::NonTerminal.new('sentence') }
70
-
71
- let(:sample_prod) do
72
- Syntax::Production.new(nt_sentence, [t_a, t_b, t_c])
73
- end
74
-
75
- let(:origin_val) { 3 }
76
- let(:dotted_rule) { Base::DottedItem.new(sample_prod, 2) }
77
- let(:complete_rule) { Base::DottedItem.new(sample_prod, 3) }
78
- let(:sample_parse_state) { ParseState.new(dotted_rule, origin_val) }
79
-
80
- # Factory method.
81
- def parse_state(origin, aDottedRule)
82
- ParseState.new(aDottedRule, origin)
83
- end
84
-
85
- it 'should render a scanning step' do
86
- # Case: token at the beginning
87
- subject.ostream.string = +''
88
- subject.trace_scanning(1, parse_state(0, dotted_rule))
89
- expectations = <<-SNIPPET
90
- |[------] . . . . .| [0:1] sentence => A B . C
91
- SNIPPET
92
- expect(output.string).to eq(expectations)
93
-
94
- # Case: token in the middle
95
- subject.ostream.string = +''
96
- subject.trace_scanning(4, sample_parse_state)
97
- expectations = <<-SNIPPET
98
- |. . . [------] . .| [3:4] sentence => A B . C
99
- SNIPPET
100
- expect(output.string).to eq(expectations)
101
-
102
- # Case: token at the end
103
- subject.ostream.string = +''
104
- subject.trace_scanning(6, parse_state(5, dotted_rule))
105
- expectations = <<-SNIPPET
106
- |. . . . . [------]| [5:6] sentence => A B . C
107
- SNIPPET
108
- expect(output.string).to eq(expectations)
109
- end
110
-
111
-
112
- it 'should render a prediction step' do
113
- # Case: initial stateset
114
- subject.ostream.string = +''
115
- subject.trace_prediction(0, parse_state(0, dotted_rule))
116
- expectations = <<-SNIPPET
117
- |> . . . . . .| [0:0] sentence => A B . C
118
- SNIPPET
119
- expect(output.string).to eq(expectations)
120
-
121
- # Case: stateset in the middle
122
- subject.ostream.string = +''
123
- subject.trace_prediction(3, sample_parse_state)
124
- expectations = <<-SNIPPET
125
- |. . . > . . .| [3:3] sentence => A B . C
126
- SNIPPET
127
- expect(output.string).to eq(expectations)
128
-
129
- # Case: final stateset
130
- subject.ostream.string = +''
131
- subject.trace_prediction(6, parse_state(6, dotted_rule))
132
- expectations = <<-SNIPPET
133
- |. . . . . . >| [6:6] sentence => A B . C
134
- SNIPPET
135
- expect(output.string).to eq(expectations)
136
- end
137
-
138
- it 'should render a completion step' do
139
- # Case: full parse completed
140
- subject.ostream.string = +''
141
- subject.trace_completion(6, parse_state(0, complete_rule))
142
- expectations = <<-SNIPPET
143
- |[=========================================]| [0:6] sentence => A B C .
144
- SNIPPET
145
- expect(output.string).to eq(expectations)
146
-
147
- # Case: step at the start (complete)
148
- subject.ostream.string = +''
149
- subject.trace_completion(1, parse_state(0, complete_rule))
150
- expectations = <<-SNIPPET
151
- |[------] . . . . .| [0:1] sentence => A B C .
152
- SNIPPET
153
- expect(output.string).to eq(expectations)
154
-
155
- # Case: step at the start (not complete)
156
- subject.ostream.string = +''
157
- subject.trace_completion(1, parse_state(0, dotted_rule))
158
- expectations = <<-SNIPPET
159
- |[------> . . . . .| [0:1] sentence => A B . C
160
- SNIPPET
161
- expect(output.string).to eq(expectations)
162
-
163
- # Case: step at the middle (complete)
164
- subject.ostream.string = +''
165
- subject.trace_completion(4, parse_state(2, complete_rule))
166
- expectations = <<-SNIPPET
167
- |. . [-------------] . .| [2:4] sentence => A B C .
168
- SNIPPET
169
- expect(output.string).to eq(expectations)
170
-
171
- # Case: step at the middle (not complete)
172
- subject.ostream.string = +''
173
- subject.trace_completion(4, parse_state(2, dotted_rule))
174
- expectations = <<-SNIPPET
175
- |. . [-------------> . .| [2:4] sentence => A B . C
176
- SNIPPET
177
- expect(output.string).to eq(expectations)
178
-
179
- # Case: step at the end (complete)
180
- subject.ostream.string = +''
181
- subject.trace_completion(6, parse_state(3, complete_rule))
182
- expectations = <<-SNIPPET
183
- |. . . [--------------------]| [3:6] sentence => A B C .
184
- SNIPPET
185
- expect(output.string).to eq(expectations)
186
-
187
- # Case: step at the end (not complete)
188
- subject.ostream.string = +''
189
- subject.trace_completion(6, parse_state(3, dotted_rule))
190
- expectations = <<-SNIPPET
191
- |. . . [-------------------->| [3:6] sentence => A B . C
192
- SNIPPET
193
- expect(output.string).to eq(expectations)
194
- end
195
- end # context
196
- end # describe
197
- end # module
198
- end # module
199
-
200
- # End of file
@@ -1,130 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../../spec_helper'
4
-
5
- require_relative '../../../lib/rley/parser/parse_state'
6
-
7
- # Load the class under test
8
- require_relative '../../../lib/rley/parser/state_set'
9
-
10
- module Rley # Open this namespace to avoid module qualifier prefixes
11
- module Parser # Open this namespace to avoid module qualifier prefixes
12
- describe StateSet do
13
- let(:dotted_rule1) { double('fake_dotted_rule1') }
14
- let(:state1) { ParseState.new(dotted_rule1, 2) }
15
- let(:dotted_rule2) { double('fake_dotted_rule2') }
16
- let(:state2) { ParseState.new(dotted_rule2, 5) }
17
-
18
- context 'Initialization:' do
19
- it 'should be created without argument' do
20
- expect { StateSet.new }.not_to raise_error
21
- end
22
-
23
- it 'should be empty at creation' do
24
- expect(subject.states).to be_empty
25
- end
26
- end # context
27
-
28
- context 'Provided services:' do
29
- it 'should push a state' do
30
- expect(subject.states).to be_empty
31
- expect { subject.push_state(state1) }.not_to raise_error
32
- expect(subject).not_to be_empty
33
- subject.push_state(state2)
34
- expect(subject.states).to eq([state1, state2])
35
- end
36
-
37
- it 'should ignore a second push of a state' do
38
- expect(subject.states).to be_empty
39
- subject.push_state(state1)
40
- subject.push_state(state2)
41
- expect(subject.states).to eq([state1, state2])
42
-
43
- # One tries to push an already pushed state
44
- expect(subject.push_state(state1)).to be_falsy
45
-
46
- # ...It is not added
47
- expect(subject.states).to eq([state1, state2])
48
- end
49
-
50
- it 'should list the states expecting a given terminal' do
51
- # Case of no state
52
- expect(subject.states_expecting(:a)).to be_empty
53
-
54
- # Adding states
55
- subject.push_state(state1)
56
- subject.push_state(state2)
57
- allow(dotted_rule1).to receive(:next_symbol).and_return(:b)
58
- allow(dotted_rule2).to receive(:next_symbol).and_return(:a)
59
- expect(subject.states_expecting(:a)).to eq([state2])
60
- expect(subject.states_expecting(:b)).to eq([state1])
61
- end
62
-
63
- it 'should list the states related to a production' do
64
- a_prod = double('fake-production')
65
-
66
- # Case of no state
67
- expect(subject.states_for(a_prod)).to be_empty
68
-
69
- # Adding states
70
- subject.push_state(state1)
71
- subject.push_state(state2)
72
- expect(dotted_rule1).to receive(:production).and_return(:dummy)
73
- expect(dotted_rule2).to receive(:production).and_return(a_prod)
74
- expect(subject.states_for(a_prod)).to eq([state2])
75
- end
76
-
77
- it 'should list the states that rewrite a given non-terminal' do
78
- non_term = double('fake-non-terminal')
79
- prod1 = double('fake-production1')
80
- prod2 = double('fake-production2')
81
-
82
- # Adding states
83
- subject.push_state(state1)
84
- subject.push_state(state2)
85
- expect(dotted_rule1).to receive(:production).and_return(prod1)
86
- expect(prod1).to receive(:lhs).and_return(:dummy)
87
- expect(dotted_rule2).to receive(:production).and_return(prod2)
88
- expect(dotted_rule2).to receive(:reduce_item?).and_return(true)
89
- expect(prod2).to receive(:lhs).and_return(non_term)
90
- expect(subject.states_rewriting(non_term)).to eq([state2])
91
- end
92
-
93
- it 'should list of ambiguous states' do
94
- prod1 = double('fake-production1')
95
- prod2 = double('fake-production2')
96
- expect(subject.ambiguities.size).to eq(0)
97
-
98
- # Adding states
99
- subject.push_state(state1)
100
- allow(dotted_rule1).to receive(:production).and_return(prod1)
101
- allow(dotted_rule1).to receive(:reduce_item?).and_return(true)
102
- allow(dotted_rule1).to receive(:lhs).and_return(:something)
103
- expect(subject.ambiguities.size).to eq(0)
104
- allow(dotted_rule2).to receive(:production).and_return(prod2)
105
- allow(dotted_rule2).to receive(:reduce_item?).and_return(true)
106
- allow(dotted_rule2).to receive(:lhs).and_return(:something_else)
107
- subject.push_state(state2)
108
- expect(subject.ambiguities.size).to eq(0)
109
- dotted_rule3 = double('fake_dotted_rule3')
110
- allow(dotted_rule3).to receive(:production).and_return(prod2)
111
- allow(dotted_rule3).to receive(:reduce_item?).and_return(true)
112
- allow(dotted_rule3).to receive(:lhs).and_return(:something_else)
113
- state3 = ParseState.new(dotted_rule3, 5)
114
- subject.push_state(state3)
115
- expect(subject.ambiguities[0]).to eq([state2, state3])
116
- end
117
-
118
- it 'should complain when impossible predecessor of parse state' do
119
- subject.push_state(state1)
120
- subject.push_state(state2)
121
- expect(dotted_rule1).to receive(:prev_position).and_return(nil)
122
- err = StandardError
123
- expect { subject.predecessor_state(state1) }.to raise_error(err)
124
- end
125
- end # context
126
- end # describe
127
- end # module
128
- end # module
129
-
130
- # End of file