rley 0.7.08 → 0.8.03
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +29 -5
- data/CHANGELOG.md +28 -4
- data/README.md +4 -5
- data/examples/NLP/nano_eng/nano_en_demo.rb +7 -11
- data/examples/NLP/nano_eng/nano_grammar.rb +18 -18
- data/examples/data_formats/JSON/json_ast_builder.rb +9 -18
- data/examples/data_formats/JSON/json_demo.rb +1 -2
- data/examples/data_formats/JSON/json_grammar.rb +11 -11
- data/examples/general/calc_iter1/calc_grammar.rb +5 -4
- data/examples/general/calc_iter2/calc_grammar.rb +9 -9
- data/examples/general/left.rb +1 -1
- data/examples/general/right.rb +1 -1
- data/lib/rley/base/dotted_item.rb +5 -0
- data/lib/rley/base/grm_items_builder.rb +6 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/engine.rb +2 -2
- data/lib/rley/interface.rb +16 -0
- data/lib/rley/notation/all_notation_nodes.rb +4 -0
- data/lib/rley/notation/ast_builder.rb +185 -0
- data/lib/rley/notation/ast_node.rb +44 -0
- data/lib/rley/notation/ast_visitor.rb +115 -0
- data/lib/rley/notation/grammar.rb +49 -0
- data/lib/rley/notation/grammar_builder.rb +505 -0
- data/lib/rley/notation/grouping_node.rb +23 -0
- data/lib/rley/notation/parser.rb +56 -0
- data/lib/rley/notation/sequence_node.rb +35 -0
- data/lib/rley/notation/symbol_node.rb +29 -0
- data/lib/rley/notation/tokenizer.rb +180 -0
- data/lib/rley/parse_rep/ast_base_builder.rb +44 -0
- data/lib/rley/parser/gfg_chart.rb +101 -6
- data/lib/rley/parser/gfg_earley_parser.rb +1 -1
- data/lib/rley/parser/gfg_parsing.rb +5 -3
- data/lib/rley/parser/parse_entry_set.rb +1 -1
- data/lib/rley/syntax/{grammar_builder.rb → base_grammar_builder.rb} +53 -15
- data/lib/rley/syntax/grm_symbol.rb +1 -1
- data/lib/rley/syntax/match_closest.rb +43 -0
- data/lib/rley/syntax/production.rb +6 -0
- data/lib/rley.rb +1 -1
- data/spec/rley/engine_spec.rb +6 -6
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -2
- data/spec/rley/notation/grammar_builder_spec.rb +302 -0
- data/spec/rley/notation/parser_spec.rb +183 -0
- data/spec/rley/notation/tokenizer_spec.rb +364 -0
- data/spec/rley/parse_rep/ast_builder_spec.rb +0 -1
- data/spec/rley/parse_rep/groucho_spec.rb +1 -1
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +1 -1
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +2 -2
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +1 -1
- data/spec/rley/parser/dangling_else_spec.rb +447 -0
- data/spec/rley/parser/gfg_earley_parser_spec.rb +118 -10
- data/spec/rley/parser/gfg_parsing_spec.rb +2 -1
- data/spec/rley/parser/parse_walker_factory_spec.rb +2 -2
- data/spec/rley/support/ambiguous_grammar_helper.rb +2 -2
- data/spec/rley/support/grammar_abc_helper.rb +2 -2
- data/spec/rley/support/grammar_ambig01_helper.rb +2 -2
- data/spec/rley/support/grammar_arr_int_helper.rb +2 -2
- data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
- data/spec/rley/support/grammar_int_seq_helper.rb +51 -0
- data/spec/rley/support/grammar_l0_helper.rb +2 -2
- data/spec/rley/support/grammar_pb_helper.rb +2 -2
- data/spec/rley/support/grammar_sppf_helper.rb +2 -2
- data/spec/rley/syntax/{grammar_builder_spec.rb → base_grammar_builder_spec.rb} +29 -11
- data/spec/rley/syntax/match_closest_spec.rb +46 -0
- data/spec/rley/syntax/production_spec.rb +4 -0
- metadata +29 -14
- data/lib/rley/parser/parse_state.rb +0 -78
- data/lib/rley/parser/parse_state_tracker.rb +0 -59
- data/lib/rley/parser/state_set.rb +0 -100
- data/spec/rley/parser/parse_state_spec.rb +0 -125
- data/spec/rley/parser/parse_tracer_spec.rb +0 -200
- data/spec/rley/parser/state_set_spec.rb +0 -130
@@ -1,200 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative '../../spec_helper'
|
4
|
-
require 'stringio'
|
5
|
-
|
6
|
-
require_relative '../../../lib/rley/syntax/terminal'
|
7
|
-
require_relative '../../../lib/rley/syntax/non_terminal'
|
8
|
-
require_relative '../../../lib/rley/syntax/production'
|
9
|
-
require_relative '../../../lib/rley/base/dotted_item'
|
10
|
-
require_relative '../../../lib/rley/parser/parse_state'
|
11
|
-
require_relative '../../../lib/rley/lexical/token'
|
12
|
-
|
13
|
-
# Load the class under test
|
14
|
-
require_relative '../../../lib/rley/parser/parse_tracer'
|
15
|
-
|
16
|
-
module Rley # Open this namespace to avoid module qualifier prefixes
|
17
|
-
module Parser # Open this namespace to avoid module qualifier prefixes
|
18
|
-
describe ParseTracer do
|
19
|
-
let(:output) { StringIO.new(+'', 'w') }
|
20
|
-
let(:tpos) { Lexical::Position.new(3, 4) }
|
21
|
-
|
22
|
-
let(:token_seq) do
|
23
|
-
literals = %w[I saw John with a dog]
|
24
|
-
literals.map do |lexeme|
|
25
|
-
Lexical::Token.new(lexeme, double('fake-terminal'), tpos)
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
subject { ParseTracer.new(1, output, token_seq) }
|
30
|
-
|
31
|
-
context 'Creation & initialization:' do
|
32
|
-
it 'should accept trace level 0' do
|
33
|
-
expect { ParseTracer.new(0, output, token_seq) }.not_to raise_error
|
34
|
-
expect(output.string).to eq('')
|
35
|
-
end
|
36
|
-
|
37
|
-
|
38
|
-
it 'should accept trace level 1' do
|
39
|
-
expect { ParseTracer.new(1, output, token_seq) }.not_to raise_error
|
40
|
-
expectations = <<-SNIPPET
|
41
|
-
['I', 'saw', 'John', 'with', 'a', 'dog']
|
42
|
-
|. I . saw . John . with . a . dog .|
|
43
|
-
SNIPPET
|
44
|
-
expect(output.string).to eq(expectations)
|
45
|
-
end
|
46
|
-
|
47
|
-
it 'should accept trace level 2' do
|
48
|
-
expect { ParseTracer.new(2, output, token_seq) }.not_to raise_error
|
49
|
-
expectations = <<-SNIPPET
|
50
|
-
['I', 'saw', 'John', 'with', 'a', 'dog']
|
51
|
-
|. I . saw . John . with . a . dog .|
|
52
|
-
SNIPPET
|
53
|
-
expect(output.string).to eq(expectations)
|
54
|
-
end
|
55
|
-
|
56
|
-
it 'should know the trace level' do
|
57
|
-
expect(subject.level).to eq(1)
|
58
|
-
end
|
59
|
-
|
60
|
-
it 'should know the output stream' do
|
61
|
-
expect(subject.ostream).to eq(output)
|
62
|
-
end
|
63
|
-
end # context
|
64
|
-
|
65
|
-
context 'Provided services:' do
|
66
|
-
let(:t_a) { Syntax::Terminal.new('A') }
|
67
|
-
let(:t_b) { Syntax::Terminal.new('B') }
|
68
|
-
let(:t_c) { Syntax::Terminal.new('C') }
|
69
|
-
let(:nt_sentence) { Syntax::NonTerminal.new('sentence') }
|
70
|
-
|
71
|
-
let(:sample_prod) do
|
72
|
-
Syntax::Production.new(nt_sentence, [t_a, t_b, t_c])
|
73
|
-
end
|
74
|
-
|
75
|
-
let(:origin_val) { 3 }
|
76
|
-
let(:dotted_rule) { Base::DottedItem.new(sample_prod, 2) }
|
77
|
-
let(:complete_rule) { Base::DottedItem.new(sample_prod, 3) }
|
78
|
-
let(:sample_parse_state) { ParseState.new(dotted_rule, origin_val) }
|
79
|
-
|
80
|
-
# Factory method.
|
81
|
-
def parse_state(origin, aDottedRule)
|
82
|
-
ParseState.new(aDottedRule, origin)
|
83
|
-
end
|
84
|
-
|
85
|
-
it 'should render a scanning step' do
|
86
|
-
# Case: token at the beginning
|
87
|
-
subject.ostream.string = +''
|
88
|
-
subject.trace_scanning(1, parse_state(0, dotted_rule))
|
89
|
-
expectations = <<-SNIPPET
|
90
|
-
|[------] . . . . .| [0:1] sentence => A B . C
|
91
|
-
SNIPPET
|
92
|
-
expect(output.string).to eq(expectations)
|
93
|
-
|
94
|
-
# Case: token in the middle
|
95
|
-
subject.ostream.string = +''
|
96
|
-
subject.trace_scanning(4, sample_parse_state)
|
97
|
-
expectations = <<-SNIPPET
|
98
|
-
|. . . [------] . .| [3:4] sentence => A B . C
|
99
|
-
SNIPPET
|
100
|
-
expect(output.string).to eq(expectations)
|
101
|
-
|
102
|
-
# Case: token at the end
|
103
|
-
subject.ostream.string = +''
|
104
|
-
subject.trace_scanning(6, parse_state(5, dotted_rule))
|
105
|
-
expectations = <<-SNIPPET
|
106
|
-
|. . . . . [------]| [5:6] sentence => A B . C
|
107
|
-
SNIPPET
|
108
|
-
expect(output.string).to eq(expectations)
|
109
|
-
end
|
110
|
-
|
111
|
-
|
112
|
-
it 'should render a prediction step' do
|
113
|
-
# Case: initial stateset
|
114
|
-
subject.ostream.string = +''
|
115
|
-
subject.trace_prediction(0, parse_state(0, dotted_rule))
|
116
|
-
expectations = <<-SNIPPET
|
117
|
-
|> . . . . . .| [0:0] sentence => A B . C
|
118
|
-
SNIPPET
|
119
|
-
expect(output.string).to eq(expectations)
|
120
|
-
|
121
|
-
# Case: stateset in the middle
|
122
|
-
subject.ostream.string = +''
|
123
|
-
subject.trace_prediction(3, sample_parse_state)
|
124
|
-
expectations = <<-SNIPPET
|
125
|
-
|. . . > . . .| [3:3] sentence => A B . C
|
126
|
-
SNIPPET
|
127
|
-
expect(output.string).to eq(expectations)
|
128
|
-
|
129
|
-
# Case: final stateset
|
130
|
-
subject.ostream.string = +''
|
131
|
-
subject.trace_prediction(6, parse_state(6, dotted_rule))
|
132
|
-
expectations = <<-SNIPPET
|
133
|
-
|. . . . . . >| [6:6] sentence => A B . C
|
134
|
-
SNIPPET
|
135
|
-
expect(output.string).to eq(expectations)
|
136
|
-
end
|
137
|
-
|
138
|
-
it 'should render a completion step' do
|
139
|
-
# Case: full parse completed
|
140
|
-
subject.ostream.string = +''
|
141
|
-
subject.trace_completion(6, parse_state(0, complete_rule))
|
142
|
-
expectations = <<-SNIPPET
|
143
|
-
|[=========================================]| [0:6] sentence => A B C .
|
144
|
-
SNIPPET
|
145
|
-
expect(output.string).to eq(expectations)
|
146
|
-
|
147
|
-
# Case: step at the start (complete)
|
148
|
-
subject.ostream.string = +''
|
149
|
-
subject.trace_completion(1, parse_state(0, complete_rule))
|
150
|
-
expectations = <<-SNIPPET
|
151
|
-
|[------] . . . . .| [0:1] sentence => A B C .
|
152
|
-
SNIPPET
|
153
|
-
expect(output.string).to eq(expectations)
|
154
|
-
|
155
|
-
# Case: step at the start (not complete)
|
156
|
-
subject.ostream.string = +''
|
157
|
-
subject.trace_completion(1, parse_state(0, dotted_rule))
|
158
|
-
expectations = <<-SNIPPET
|
159
|
-
|[------> . . . . .| [0:1] sentence => A B . C
|
160
|
-
SNIPPET
|
161
|
-
expect(output.string).to eq(expectations)
|
162
|
-
|
163
|
-
# Case: step at the middle (complete)
|
164
|
-
subject.ostream.string = +''
|
165
|
-
subject.trace_completion(4, parse_state(2, complete_rule))
|
166
|
-
expectations = <<-SNIPPET
|
167
|
-
|. . [-------------] . .| [2:4] sentence => A B C .
|
168
|
-
SNIPPET
|
169
|
-
expect(output.string).to eq(expectations)
|
170
|
-
|
171
|
-
# Case: step at the middle (not complete)
|
172
|
-
subject.ostream.string = +''
|
173
|
-
subject.trace_completion(4, parse_state(2, dotted_rule))
|
174
|
-
expectations = <<-SNIPPET
|
175
|
-
|. . [-------------> . .| [2:4] sentence => A B . C
|
176
|
-
SNIPPET
|
177
|
-
expect(output.string).to eq(expectations)
|
178
|
-
|
179
|
-
# Case: step at the end (complete)
|
180
|
-
subject.ostream.string = +''
|
181
|
-
subject.trace_completion(6, parse_state(3, complete_rule))
|
182
|
-
expectations = <<-SNIPPET
|
183
|
-
|. . . [--------------------]| [3:6] sentence => A B C .
|
184
|
-
SNIPPET
|
185
|
-
expect(output.string).to eq(expectations)
|
186
|
-
|
187
|
-
# Case: step at the end (not complete)
|
188
|
-
subject.ostream.string = +''
|
189
|
-
subject.trace_completion(6, parse_state(3, dotted_rule))
|
190
|
-
expectations = <<-SNIPPET
|
191
|
-
|. . . [-------------------->| [3:6] sentence => A B . C
|
192
|
-
SNIPPET
|
193
|
-
expect(output.string).to eq(expectations)
|
194
|
-
end
|
195
|
-
end # context
|
196
|
-
end # describe
|
197
|
-
end # module
|
198
|
-
end # module
|
199
|
-
|
200
|
-
# End of file
|
@@ -1,130 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative '../../spec_helper'
|
4
|
-
|
5
|
-
require_relative '../../../lib/rley/parser/parse_state'
|
6
|
-
|
7
|
-
# Load the class under test
|
8
|
-
require_relative '../../../lib/rley/parser/state_set'
|
9
|
-
|
10
|
-
module Rley # Open this namespace to avoid module qualifier prefixes
|
11
|
-
module Parser # Open this namespace to avoid module qualifier prefixes
|
12
|
-
describe StateSet do
|
13
|
-
let(:dotted_rule1) { double('fake_dotted_rule1') }
|
14
|
-
let(:state1) { ParseState.new(dotted_rule1, 2) }
|
15
|
-
let(:dotted_rule2) { double('fake_dotted_rule2') }
|
16
|
-
let(:state2) { ParseState.new(dotted_rule2, 5) }
|
17
|
-
|
18
|
-
context 'Initialization:' do
|
19
|
-
it 'should be created without argument' do
|
20
|
-
expect { StateSet.new }.not_to raise_error
|
21
|
-
end
|
22
|
-
|
23
|
-
it 'should be empty at creation' do
|
24
|
-
expect(subject.states).to be_empty
|
25
|
-
end
|
26
|
-
end # context
|
27
|
-
|
28
|
-
context 'Provided services:' do
|
29
|
-
it 'should push a state' do
|
30
|
-
expect(subject.states).to be_empty
|
31
|
-
expect { subject.push_state(state1) }.not_to raise_error
|
32
|
-
expect(subject).not_to be_empty
|
33
|
-
subject.push_state(state2)
|
34
|
-
expect(subject.states).to eq([state1, state2])
|
35
|
-
end
|
36
|
-
|
37
|
-
it 'should ignore a second push of a state' do
|
38
|
-
expect(subject.states).to be_empty
|
39
|
-
subject.push_state(state1)
|
40
|
-
subject.push_state(state2)
|
41
|
-
expect(subject.states).to eq([state1, state2])
|
42
|
-
|
43
|
-
# One tries to push an already pushed state
|
44
|
-
expect(subject.push_state(state1)).to be_falsy
|
45
|
-
|
46
|
-
# ...It is not added
|
47
|
-
expect(subject.states).to eq([state1, state2])
|
48
|
-
end
|
49
|
-
|
50
|
-
it 'should list the states expecting a given terminal' do
|
51
|
-
# Case of no state
|
52
|
-
expect(subject.states_expecting(:a)).to be_empty
|
53
|
-
|
54
|
-
# Adding states
|
55
|
-
subject.push_state(state1)
|
56
|
-
subject.push_state(state2)
|
57
|
-
allow(dotted_rule1).to receive(:next_symbol).and_return(:b)
|
58
|
-
allow(dotted_rule2).to receive(:next_symbol).and_return(:a)
|
59
|
-
expect(subject.states_expecting(:a)).to eq([state2])
|
60
|
-
expect(subject.states_expecting(:b)).to eq([state1])
|
61
|
-
end
|
62
|
-
|
63
|
-
it 'should list the states related to a production' do
|
64
|
-
a_prod = double('fake-production')
|
65
|
-
|
66
|
-
# Case of no state
|
67
|
-
expect(subject.states_for(a_prod)).to be_empty
|
68
|
-
|
69
|
-
# Adding states
|
70
|
-
subject.push_state(state1)
|
71
|
-
subject.push_state(state2)
|
72
|
-
expect(dotted_rule1).to receive(:production).and_return(:dummy)
|
73
|
-
expect(dotted_rule2).to receive(:production).and_return(a_prod)
|
74
|
-
expect(subject.states_for(a_prod)).to eq([state2])
|
75
|
-
end
|
76
|
-
|
77
|
-
it 'should list the states that rewrite a given non-terminal' do
|
78
|
-
non_term = double('fake-non-terminal')
|
79
|
-
prod1 = double('fake-production1')
|
80
|
-
prod2 = double('fake-production2')
|
81
|
-
|
82
|
-
# Adding states
|
83
|
-
subject.push_state(state1)
|
84
|
-
subject.push_state(state2)
|
85
|
-
expect(dotted_rule1).to receive(:production).and_return(prod1)
|
86
|
-
expect(prod1).to receive(:lhs).and_return(:dummy)
|
87
|
-
expect(dotted_rule2).to receive(:production).and_return(prod2)
|
88
|
-
expect(dotted_rule2).to receive(:reduce_item?).and_return(true)
|
89
|
-
expect(prod2).to receive(:lhs).and_return(non_term)
|
90
|
-
expect(subject.states_rewriting(non_term)).to eq([state2])
|
91
|
-
end
|
92
|
-
|
93
|
-
it 'should list of ambiguous states' do
|
94
|
-
prod1 = double('fake-production1')
|
95
|
-
prod2 = double('fake-production2')
|
96
|
-
expect(subject.ambiguities.size).to eq(0)
|
97
|
-
|
98
|
-
# Adding states
|
99
|
-
subject.push_state(state1)
|
100
|
-
allow(dotted_rule1).to receive(:production).and_return(prod1)
|
101
|
-
allow(dotted_rule1).to receive(:reduce_item?).and_return(true)
|
102
|
-
allow(dotted_rule1).to receive(:lhs).and_return(:something)
|
103
|
-
expect(subject.ambiguities.size).to eq(0)
|
104
|
-
allow(dotted_rule2).to receive(:production).and_return(prod2)
|
105
|
-
allow(dotted_rule2).to receive(:reduce_item?).and_return(true)
|
106
|
-
allow(dotted_rule2).to receive(:lhs).and_return(:something_else)
|
107
|
-
subject.push_state(state2)
|
108
|
-
expect(subject.ambiguities.size).to eq(0)
|
109
|
-
dotted_rule3 = double('fake_dotted_rule3')
|
110
|
-
allow(dotted_rule3).to receive(:production).and_return(prod2)
|
111
|
-
allow(dotted_rule3).to receive(:reduce_item?).and_return(true)
|
112
|
-
allow(dotted_rule3).to receive(:lhs).and_return(:something_else)
|
113
|
-
state3 = ParseState.new(dotted_rule3, 5)
|
114
|
-
subject.push_state(state3)
|
115
|
-
expect(subject.ambiguities[0]).to eq([state2, state3])
|
116
|
-
end
|
117
|
-
|
118
|
-
it 'should complain when impossible predecessor of parse state' do
|
119
|
-
subject.push_state(state1)
|
120
|
-
subject.push_state(state2)
|
121
|
-
expect(dotted_rule1).to receive(:prev_position).and_return(nil)
|
122
|
-
err = StandardError
|
123
|
-
expect { subject.predecessor_state(state1) }.to raise_error(err)
|
124
|
-
end
|
125
|
-
end # context
|
126
|
-
end # describe
|
127
|
-
end # module
|
128
|
-
end # module
|
129
|
-
|
130
|
-
# End of file
|