rley 0.7.03 → 0.7.08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +362 -62
- data/.travis.yml +6 -7
- data/CHANGELOG.md +20 -1
- data/LICENSE.txt +1 -1
- data/README.md +6 -7
- data/Rakefile +2 -0
- data/appveyor.yml +2 -4
- data/examples/NLP/benchmark_pico_en.rb +2 -0
- data/examples/NLP/engtagger.rb +193 -188
- data/examples/NLP/nano_eng/nano_en_demo.rb +2 -0
- data/examples/NLP/nano_eng/nano_grammar.rb +7 -5
- data/examples/NLP/pico_en_demo.rb +2 -0
- data/examples/data_formats/JSON/cli_options.rb +3 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +14 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +14 -21
- data/examples/data_formats/JSON/json_demo.rb +2 -0
- data/examples/data_formats/JSON/json_grammar.rb +4 -2
- data/examples/data_formats/JSON/json_lexer.rb +10 -8
- data/examples/data_formats/JSON/json_minifier.rb +3 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +15 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +25 -37
- data/examples/general/calc_iter1/calc_demo.rb +2 -0
- data/examples/general/calc_iter1/calc_grammar.rb +4 -2
- data/examples/general/calc_iter1/calc_lexer.rb +8 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +7 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +7 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +29 -43
- data/examples/general/calc_iter2/calc_demo.rb +2 -0
- data/examples/general/calc_iter2/calc_grammar.rb +5 -3
- data/examples/general/calc_iter2/calc_lexer.rb +13 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +28 -26
- data/examples/general/left.rb +4 -2
- data/examples/general/right.rb +4 -2
- data/lib/rley.rb +2 -0
- data/lib/rley/base/base_parser.rb +2 -0
- data/lib/rley/base/dotted_item.rb +38 -41
- data/lib/rley/base/grm_items_builder.rb +2 -0
- data/lib/rley/constants.rb +5 -3
- data/lib/rley/engine.rb +22 -24
- data/lib/rley/formatter/asciitree.rb +6 -4
- data/lib/rley/formatter/base_formatter.rb +2 -0
- data/lib/rley/formatter/bracket_notation.rb +3 -8
- data/lib/rley/formatter/debug.rb +8 -6
- data/lib/rley/formatter/json.rb +4 -2
- data/lib/rley/gfg/call_edge.rb +3 -1
- data/lib/rley/gfg/edge.rb +7 -5
- data/lib/rley/gfg/end_vertex.rb +4 -6
- data/lib/rley/gfg/epsilon_edge.rb +3 -5
- data/lib/rley/gfg/grm_flow_graph.rb +31 -25
- data/lib/rley/gfg/item_vertex.rb +12 -22
- data/lib/rley/gfg/non_terminal_vertex.rb +6 -4
- data/lib/rley/gfg/return_edge.rb +2 -0
- data/lib/rley/gfg/scan_edge.rb +3 -1
- data/lib/rley/gfg/shortcut_edge.rb +4 -2
- data/lib/rley/gfg/start_vertex.rb +6 -8
- data/lib/rley/gfg/vertex.rb +47 -41
- data/lib/rley/lexical/token.rb +3 -1
- data/lib/rley/lexical/token_range.rb +8 -6
- data/lib/rley/parse_forest_visitor.rb +7 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +11 -11
- data/lib/rley/parse_rep/cst_builder.rb +7 -4
- data/lib/rley/parse_rep/parse_forest_builder.rb +36 -25
- data/lib/rley/parse_rep/parse_forest_factory.rb +5 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +18 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +15 -15
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -25
- data/lib/rley/parse_tree_visitor.rb +3 -1
- data/lib/rley/parser/error_reason.rb +9 -8
- data/lib/rley/parser/gfg_chart.rb +54 -22
- data/lib/rley/parser/gfg_earley_parser.rb +3 -1
- data/lib/rley/parser/gfg_parsing.rb +51 -31
- data/lib/rley/parser/parse_entry.rb +29 -33
- data/lib/rley/parser/parse_entry_set.rb +32 -27
- data/lib/rley/parser/parse_entry_tracker.rb +6 -4
- data/lib/rley/parser/parse_state.rb +18 -21
- data/lib/rley/parser/parse_state_tracker.rb +6 -4
- data/lib/rley/parser/parse_tracer.rb +15 -13
- data/lib/rley/parser/parse_walker_factory.rb +28 -29
- data/lib/rley/parser/state_set.rb +11 -10
- data/lib/rley/ptree/non_terminal_node.rb +10 -6
- data/lib/rley/ptree/parse_tree.rb +6 -4
- data/lib/rley/ptree/parse_tree_node.rb +7 -5
- data/lib/rley/ptree/terminal_node.rb +9 -7
- data/lib/rley/rley_error.rb +12 -10
- data/lib/rley/sppf/alternative_node.rb +8 -6
- data/lib/rley/sppf/composite_node.rb +9 -7
- data/lib/rley/sppf/epsilon_node.rb +5 -3
- data/lib/rley/sppf/leaf_node.rb +5 -3
- data/lib/rley/sppf/non_terminal_node.rb +2 -0
- data/lib/rley/sppf/parse_forest.rb +19 -17
- data/lib/rley/sppf/sppf_node.rb +9 -8
- data/lib/rley/sppf/token_node.rb +5 -3
- data/lib/rley/syntax/grammar.rb +7 -5
- data/lib/rley/syntax/grammar_builder.rb +11 -9
- data/lib/rley/syntax/grm_symbol.rb +8 -6
- data/lib/rley/syntax/literal.rb +2 -0
- data/lib/rley/syntax/non_terminal.rb +11 -15
- data/lib/rley/syntax/production.rb +13 -11
- data/lib/rley/syntax/symbol_seq.rb +10 -10
- data/lib/rley/syntax/terminal.rb +6 -5
- data/lib/rley/syntax/verbatim_symbol.rb +5 -3
- data/lib/support/base_tokenizer.rb +23 -20
- data/spec/rley/base/dotted_item_spec.rb +4 -2
- data/spec/rley/base/grm_items_builder_spec.rb +2 -0
- data/spec/rley/engine_spec.rb +47 -9
- data/spec/rley/formatter/asciitree_spec.rb +11 -9
- data/spec/rley/formatter/bracket_notation_spec.rb +16 -14
- data/spec/rley/formatter/debug_spec.rb +4 -2
- data/spec/rley/formatter/json_spec.rb +5 -3
- data/spec/rley/gfg/call_edge_spec.rb +2 -0
- data/spec/rley/gfg/edge_spec.rb +2 -0
- data/spec/rley/gfg/end_vertex_spec.rb +7 -5
- data/spec/rley/gfg/epsilon_edge_spec.rb +2 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -0
- data/spec/rley/gfg/item_vertex_spec.rb +12 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +5 -3
- data/spec/rley/gfg/return_edge_spec.rb +2 -0
- data/spec/rley/gfg/scan_edge_spec.rb +2 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +3 -1
- data/spec/rley/gfg/start_vertex_spec.rb +7 -5
- data/spec/rley/gfg/vertex_spec.rb +5 -3
- data/spec/rley/lexical/token_range_spec.rb +18 -16
- data/spec/rley/lexical/token_spec.rb +4 -2
- data/spec/rley/parse_forest_visitor_spec.rb +167 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +46 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +8 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +7 -5
- data/spec/rley/parse_rep/groucho_spec.rb +25 -25
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +28 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +8 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +4 -2
- data/spec/rley/parse_tree_visitor_spec.rb +12 -8
- data/spec/rley/parser/error_reason_spec.rb +8 -6
- data/spec/rley/parser/gfg_chart_spec.rb +17 -4
- data/spec/rley/parser/gfg_earley_parser_spec.rb +16 -11
- data/spec/rley/parser/gfg_parsing_spec.rb +41 -252
- data/spec/rley/parser/parse_entry_set_spec.rb +2 -0
- data/spec/rley/parser/parse_entry_spec.rb +21 -19
- data/spec/rley/parser/parse_state_spec.rb +7 -5
- data/spec/rley/parser/parse_tracer_spec.rb +16 -14
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -8
- data/spec/rley/parser/state_set_spec.rb +24 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +7 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +6 -4
- data/spec/rley/ptree/parse_tree_spec.rb +2 -0
- data/spec/rley/ptree/terminal_node_spec.rb +8 -6
- data/spec/rley/sppf/alternative_node_spec.rb +8 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +5 -3
- data/spec/rley/sppf/token_node_spec.rb +6 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +5 -4
- data/spec/rley/support/expectation_helper.rb +2 -0
- data/spec/rley/support/grammar_abc_helper.rb +4 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +6 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +6 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +6 -5
- data/spec/rley/support/grammar_helper.rb +2 -0
- data/spec/rley/support/grammar_l0_helper.rb +15 -16
- data/spec/rley/support/grammar_pb_helper.rb +8 -5
- data/spec/rley/support/grammar_sppf_helper.rb +3 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +7 -5
- data/spec/rley/syntax/grammar_spec.rb +8 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +3 -1
- data/spec/rley/syntax/literal_spec.rb +2 -0
- data/spec/rley/syntax/non_terminal_spec.rb +10 -8
- data/spec/rley/syntax/production_spec.rb +15 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +4 -2
- data/spec/rley/syntax/terminal_spec.rb +7 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +3 -1
- data/spec/spec_helper.rb +2 -12
- data/spec/support/base_tokenizer_spec.rb +9 -2
- metadata +21 -63
- data/.simplecov +0 -7
- data/Gemfile +0 -8
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
@@ -23,12 +25,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
23
25
|
builder = Syntax::GrammarBuilder.new do
|
24
26
|
add_terminals('a', 'b')
|
25
27
|
rule 'Phi' => 'S'
|
26
|
-
rule 'S' =>
|
27
|
-
rule 'S' =>
|
28
|
+
rule 'S' => 'A T'
|
29
|
+
rule 'S' => 'a T'
|
28
30
|
rule 'A' => 'a'
|
29
|
-
rule 'A' =>
|
31
|
+
rule 'A' => 'B A'
|
30
32
|
rule 'B' => []
|
31
|
-
rule 'T' =>
|
33
|
+
rule 'T' => 'b b b'
|
32
34
|
end
|
33
35
|
builder.grammar
|
34
36
|
end
|
@@ -47,11 +49,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
47
49
|
end
|
48
50
|
|
49
51
|
# Emit a text representation of the current path.
|
50
|
-
def path_to_s
|
52
|
+
def path_to_s
|
51
53
|
text_parts = subject.curr_path.map do |path_element|
|
52
54
|
path_element.to_string(0)
|
53
55
|
end
|
54
|
-
|
56
|
+
text_parts.join('/')
|
55
57
|
end
|
56
58
|
|
57
59
|
context 'Initialization:' do
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
@@ -36,11 +38,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
36
38
|
end
|
37
39
|
|
38
40
|
# Emit a text representation of the current path.
|
39
|
-
def path_to_s
|
41
|
+
def path_to_s
|
40
42
|
text_parts = subject.curr_path.map do |path_element|
|
41
43
|
path_element.to_string(0)
|
42
44
|
end
|
43
|
-
|
45
|
+
text_parts.join('/')
|
44
46
|
end
|
45
47
|
|
46
48
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../lib/rley/lexical/token'
|
@@ -9,7 +11,7 @@ require_relative './support/grammar_sppf_helper'
|
|
9
11
|
require_relative '../../lib/rley/parse_tree_visitor'
|
10
12
|
|
11
13
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
12
|
-
describe ParseTreeVisitor do
|
14
|
+
describe ParseTreeVisitor do
|
13
15
|
let(:grammar_abc) do
|
14
16
|
sandbox = Object.new
|
15
17
|
sandbox.extend(GrammarABCHelper)
|
@@ -43,7 +45,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
43
45
|
# Capital letters represent non-terminal nodes
|
44
46
|
let(:grm_abc_ptree1) do
|
45
47
|
engine = Rley::Engine.new
|
46
|
-
engine.use_grammar(grammar_abc)
|
48
|
+
engine.use_grammar(grammar_abc)
|
47
49
|
parse_result = engine.parse(grm_abc_tokens1)
|
48
50
|
ptree = engine.convert(parse_result)
|
49
51
|
ptree
|
@@ -159,6 +161,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
159
161
|
subject.end_visit_ptree(grm_abc_ptree1)
|
160
162
|
end
|
161
163
|
|
164
|
+
# rubocop: disable Naming/VariableNumber
|
162
165
|
it 'should begin the visit when requested' do
|
163
166
|
# Reminder: parse tree structure is
|
164
167
|
# S[0,5]
|
@@ -207,11 +210,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
207
210
|
expectations.each do |(msg, args)|
|
208
211
|
expect(listener1).to receive(msg).with(*args).ordered
|
209
212
|
end
|
210
|
-
|
213
|
+
|
211
214
|
# Here we go...
|
212
215
|
subject.start
|
213
216
|
end
|
214
|
-
|
217
|
+
|
215
218
|
it 'should also visit in pre-order' do
|
216
219
|
# Reminder: parse tree structure is
|
217
220
|
# S[0,5]
|
@@ -226,7 +229,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
226
229
|
root = grm_abc_ptree1.root
|
227
230
|
# Here we defeat encapsulation for the good cause
|
228
231
|
subject.instance_variable_set(:@traversal, :pre_order)
|
229
|
-
|
232
|
+
|
230
233
|
children = root.subnodes
|
231
234
|
big_a_1 = children[0]
|
232
235
|
big_a_1_children = big_a_1.subnodes
|
@@ -237,7 +240,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
237
240
|
expectations = [
|
238
241
|
[:before_ptree, [grm_abc_ptree1]]
|
239
242
|
# TODO: fix this test
|
240
|
-
# [:before_subnodes, [root, children]],
|
243
|
+
# [:before_subnodes, [root, children]],
|
241
244
|
# [:before_non_terminal, [root]],
|
242
245
|
|
243
246
|
# [:before_non_terminal, [big_a_1]],
|
@@ -265,10 +268,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
265
268
|
expectations.each do |(msg, args)|
|
266
269
|
expect(listener1).to receive(msg).with(*args).ordered
|
267
270
|
end
|
268
|
-
|
271
|
+
|
269
272
|
# Here we go...
|
270
273
|
subject.start
|
271
|
-
end
|
274
|
+
end
|
275
|
+
# rubocop: enable Naming/VariableNumber
|
272
276
|
end # context
|
273
277
|
end # describe
|
274
278
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require_relative '../../../lib/rley/lexical/token'
|
3
5
|
|
@@ -44,8 +46,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
44
46
|
|
45
47
|
context 'Initialization:' do
|
46
48
|
it 'should be created with arguments' do
|
47
|
-
expect do
|
48
|
-
ExpectationNotMet.new(3, err_token, terminals)
|
49
|
+
expect do
|
50
|
+
ExpectationNotMet.new(3, err_token, terminals)
|
49
51
|
end.not_to raise_error
|
50
52
|
end
|
51
53
|
|
@@ -73,8 +75,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
73
75
|
|
74
76
|
context 'Initialization:' do
|
75
77
|
it 'should be created with arguments' do
|
76
|
-
expect do
|
77
|
-
UnexpectedToken.new(3, err_token, terminals)
|
78
|
+
expect do
|
79
|
+
UnexpectedToken.new(3, err_token, terminals)
|
78
80
|
end.not_to raise_error
|
79
81
|
end
|
80
82
|
end # context
|
@@ -105,8 +107,8 @@ MESSAGE_END
|
|
105
107
|
|
106
108
|
context 'Initialization:' do
|
107
109
|
it 'should be created with arguments' do
|
108
|
-
expect do
|
109
|
-
PrematureInputEnd.new(3, err_token, terminals)
|
110
|
+
expect do
|
111
|
+
PrematureInputEnd.new(3, err_token, terminals)
|
110
112
|
end.not_to raise_error
|
111
113
|
end
|
112
114
|
end # context
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
|
@@ -75,13 +77,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
75
77
|
|
76
78
|
context 'Provided services:' do
|
77
79
|
it 'should accept the pushing of a parse entry in existing set' do
|
78
|
-
expect(subject.sets[0].entries.size).to eq(1)
|
80
|
+
expect(subject.sets[0].entries.size).to eq(1)
|
79
81
|
subject.push_entry(second_vertex, 0, 0, :scan_rule)
|
80
82
|
expect(subject.sets[0].entries.size).to eq(2)
|
81
83
|
end
|
82
|
-
|
84
|
+
|
83
85
|
it 'should accept the pushing of a parse entry in new set' do
|
84
|
-
expect(subject.sets[0].entries.size).to eq(1)
|
86
|
+
expect(subject.sets[0].entries.size).to eq(1)
|
85
87
|
subject.push_entry(second_vertex, 0, 1, :scan_rule)
|
86
88
|
expect(subject.sets[0].entries.size).to eq(1)
|
87
89
|
expect(subject.sets.size).to eq(2)
|
@@ -90,7 +92,18 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
90
92
|
|
91
93
|
it 'should retrieve an existing set at given position' do
|
92
94
|
expect(subject[0]).to eq(subject.sets[0])
|
93
|
-
end
|
95
|
+
end
|
96
|
+
|
97
|
+
it 'should a user-friendly text representation of itself' do
|
98
|
+
subject.push_entry(second_vertex, 0, 1, :scan_rule)
|
99
|
+
representation = <<REPR
|
100
|
+
State[0]
|
101
|
+
.S | 0
|
102
|
+
State[1]
|
103
|
+
S => . A | 0
|
104
|
+
REPR
|
105
|
+
expect(subject.to_s).to eq(representation)
|
106
|
+
end
|
94
107
|
end # context
|
95
108
|
end # describe
|
96
109
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
require_relative '../../../lib/rley/syntax/verbatim_symbol'
|
@@ -53,7 +55,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
53
55
|
let(:plus) { Syntax::VerbatimSymbol.new('+') }
|
54
56
|
let(:star) { Syntax::VerbatimSymbol.new('*') }
|
55
57
|
let(:integer) do
|
56
|
-
integer_pattern = /[-+]?[0-9]+/
|
58
|
+
integer_pattern = /[-+]?[0-9]+/ # Decimal notation
|
57
59
|
Syntax::Literal.new('integer', integer_pattern)
|
58
60
|
end
|
59
61
|
let(:prod_P) { Syntax::Production.new(nt_P, [nt_S]) }
|
@@ -69,7 +71,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
69
71
|
|
70
72
|
# Helper method that mimicks the output of a tokenizer
|
71
73
|
# for the language specified by grammar_expr
|
72
|
-
def grm2_tokens
|
74
|
+
def grm2_tokens
|
73
75
|
input_sequence = [
|
74
76
|
{ '2' => 'integer' },
|
75
77
|
'+',
|
@@ -77,7 +79,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
77
79
|
'*',
|
78
80
|
{ '4' => 'integer' }
|
79
81
|
]
|
80
|
-
|
82
|
+
build_token_sequence(input_sequence, grammar_expr)
|
81
83
|
end
|
82
84
|
|
83
85
|
# Default instantiation rule
|
@@ -102,6 +104,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
102
104
|
end # context
|
103
105
|
|
104
106
|
context 'Parsing: ' do
|
107
|
+
# rubocop: disable Naming/VariableNumber
|
105
108
|
it 'should parse a valid simple input' do
|
106
109
|
parse_result = subject.parse(grm1_tokens)
|
107
110
|
expect(parse_result.success?).to eq(true)
|
@@ -149,7 +152,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
149
152
|
expected = [
|
150
153
|
'A => b . | 2', # scan 'b'
|
151
154
|
'A. | 2', # exit rule
|
152
|
-
'A => a A . c | 1'
|
155
|
+
'A => a A . c | 1' # end rule
|
153
156
|
]
|
154
157
|
entry_set_3 = parse_result.chart[3]
|
155
158
|
expect(entry_set_3.entries.size).to eq(3)
|
@@ -181,6 +184,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
181
184
|
expect(entry_set_5.entries.size).to eq(4)
|
182
185
|
compare_entry_texts(entry_set_5, expected)
|
183
186
|
end
|
187
|
+
# rubocop: enable Naming/VariableNumber
|
184
188
|
|
185
189
|
it 'should parse a valid simple expression' do
|
186
190
|
instance = GFGEarleyParser.new(grammar_expr)
|
@@ -284,10 +288,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
284
288
|
# A => ;
|
285
289
|
t_x = Syntax::VerbatimSymbol.new('x')
|
286
290
|
|
287
|
-
builder = Syntax::GrammarBuilder.new
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
+
builder = Syntax::GrammarBuilder.new do
|
292
|
+
add_terminals(t_x)
|
293
|
+
rule 'Ss' => 'A A x'
|
294
|
+
rule 'A' => []
|
295
|
+
end
|
291
296
|
pos = Lexical::Position.new(1, 1)
|
292
297
|
tokens = [Lexical::Token.new('x', t_x, pos)]
|
293
298
|
|
@@ -406,7 +411,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
406
411
|
'P => S . | 0', # end rule
|
407
412
|
"S => S . '+' S | 0", # end rule
|
408
413
|
"S => S . '*' S | 0", # end rule
|
409
|
-
'P. | 0'
|
414
|
+
'P. | 0' # exit rule
|
410
415
|
]
|
411
416
|
compare_entry_texts(parse_result.chart[3], expected)
|
412
417
|
|
@@ -549,7 +554,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
549
554
|
'E => E . + E | 2', # end rule
|
550
555
|
'S => E . | 0', # end rule
|
551
556
|
'E => E . + E | 0', # end rule
|
552
|
-
'S. | 0'
|
557
|
+
'S. | 0' # exit rule
|
553
558
|
]
|
554
559
|
compare_entry_texts(parse_result.chart[5], expected)
|
555
560
|
end
|
@@ -618,7 +623,7 @@ MSG
|
|
618
623
|
]
|
619
624
|
compare_entry_texts(parse_result.chart[2], expected)
|
620
625
|
|
621
|
-
err_msg = "Premature end of input after '+' at position line 1, "
|
626
|
+
err_msg = +"Premature end of input after '+' at position line 1, "
|
622
627
|
err_msg << 'column 3'
|
623
628
|
err_msg << "\nExpected one of: ['int', '(']."
|
624
629
|
expect(parse_result.failure_reason.message).to eq(err_msg)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
|
@@ -25,7 +27,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
25
27
|
include GrammarABCHelper # Mix-in module with builder for grammar abc
|
26
28
|
include GrammarBExprHelper # Mix-in with builder for simple expressions
|
27
29
|
include GrammarHelper # Mix-in with method for creating token sequence
|
28
|
-
|
30
|
+
|
29
31
|
# Helper method. Create an array of dotted items
|
30
32
|
# from the given grammar
|
31
33
|
def build_items_for_grammar(aGrammar)
|
@@ -93,14 +95,14 @@ SNIPPET
|
|
93
95
|
|
94
96
|
context 'Parsing:' do
|
95
97
|
# Utility method to fill the first entry set...
|
96
|
-
def fill_first_set
|
98
|
+
def fill_first_set
|
97
99
|
subject.start_rule(subject.initial_entry, 0)
|
98
100
|
subject.call_rule(subject.chart[0].last, 0)
|
99
101
|
subject.start_rule(subject.chart[0].last, 0)
|
100
102
|
end
|
101
103
|
|
102
104
|
# Utility method to initialize the second entry set...
|
103
|
-
def seed_second_set
|
105
|
+
def seed_second_set
|
104
106
|
# Cheating: we change the tokens to scan...
|
105
107
|
# Seeding second entry set...
|
106
108
|
subject.scan_rule(0, grm1_token_b[0])
|
@@ -164,7 +166,7 @@ SNIPPET
|
|
164
166
|
expect(subject.chart[0].size).to eq(3)
|
165
167
|
# Last entry is: (.A, 0)
|
166
168
|
dot_A_entry = subject.chart[0].last
|
167
|
-
|
169
|
+
|
168
170
|
subject.start_rule(dot_A_entry, 0)
|
169
171
|
|
170
172
|
# Expectations: two entries:
|
@@ -194,7 +196,7 @@ SNIPPET
|
|
194
196
|
expect(last_entry.vertex.label).to eq('A => a . A c')
|
195
197
|
expect(last_entry.origin).to eq(0)
|
196
198
|
antecedence = subject.antecedence
|
197
|
-
expect(antecedence.fetch(last_entry)).to eq([fourth_entry])
|
199
|
+
expect(antecedence.fetch(last_entry)).to eq([fourth_entry])
|
198
200
|
end
|
199
201
|
|
200
202
|
it 'should apply the exit rule correctly' do
|
@@ -219,7 +221,7 @@ SNIPPET
|
|
219
221
|
exit_entry = subject.chart[1].last
|
220
222
|
expect(exit_entry.vertex.label).to eq('A.')
|
221
223
|
expect(exit_entry.origin).to eq(0)
|
222
|
-
expect(subject.antecedence.fetch(exit_entry)).to eq([last_entry])
|
224
|
+
expect(subject.antecedence.fetch(exit_entry)).to eq([last_entry])
|
223
225
|
end
|
224
226
|
|
225
227
|
it 'should apply the end rule correctly' do
|
@@ -248,10 +250,8 @@ SNIPPET
|
|
248
250
|
expect(end_entry.origin).to eq(0)
|
249
251
|
expect(subject.antecedence.fetch(end_entry)).to eq([exit_entry])
|
250
252
|
end
|
251
|
-
=begin
|
252
|
-
|
253
|
-
|
254
253
|
|
254
|
+
=begin
|
255
255
|
it 'should retrieve the parse states that expect a given terminal' do
|
256
256
|
item1 = DottedItem.new(prod_A1, 2)
|
257
257
|
item2 = DottedItem.new(prod_A1, 1)
|
@@ -282,18 +282,7 @@ SNIPPET
|
|
282
282
|
=end
|
283
283
|
end # context
|
284
284
|
|
285
|
-
context '
|
286
|
-
let(:sample_grammar1) do
|
287
|
-
builder = grammar_abc_builder
|
288
|
-
builder.grammar
|
289
|
-
end
|
290
|
-
|
291
|
-
let(:token_seq1) do
|
292
|
-
%w[a a b c c].map do |letter|
|
293
|
-
Lexical::Token.new(letter, sample_grammar1.name2symbol[letter])
|
294
|
-
end
|
295
|
-
end
|
296
|
-
|
285
|
+
context 'Provided services:' do
|
297
286
|
let(:b_expr_grammar) do
|
298
287
|
builder = grammar_expr_builder
|
299
288
|
builder.grammar
|
@@ -308,247 +297,47 @@ SNIPPET
|
|
308
297
|
tokens = expr_tokenizer('2 + 3 * 4')
|
309
298
|
parser.parse(tokens)
|
310
299
|
end
|
311
|
-
|
312
|
-
it 'should indicate whether a parse succeeded' do
|
313
|
-
expect(subject.success?).to be_truthy
|
314
|
-
end
|
315
|
-
|
316
|
-
it 'should build a parse forest' do
|
317
|
-
expect { subject.parse_forest }.not_to raise_error if subject.success?
|
318
|
-
end
|
319
|
-
=begin
|
320
|
-
it 'should create the root of a parse forest' do
|
321
|
-
(entry_tracker, builder) = prepare_parse_forest(subject)
|
322
|
-
# The root node should correspond to the start symbol and
|
323
|
-
# its direct children should correspond to rhs of start production
|
324
|
-
expected_text = <<-SNIPPET
|
325
|
-
P[0, 5]
|
326
|
-
+- S[0, 5]
|
327
|
-
SNIPPET
|
328
|
-
root_text = builder.root.to_string(0)
|
329
|
-
expect(root_text).to eq(expected_text.chomp)
|
330
|
-
|
331
|
-
expect(entry_tracker.entry_set_index).to eq(subject.tokens.size)
|
332
|
-
expected_entry = 'P => S . | 0'
|
333
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
334
|
-
expect(builder.current_node.to_string(0)).to eq('S[0, 5]')
|
335
|
-
end
|
336
|
-
=end
|
337
|
-
=begin
|
338
|
-
it 'should use a reduce item for a matched non-terminal' do
|
339
|
-
# Setup
|
340
|
-
(entry_tracker, builder) = prepare_parse_tree(subject)
|
341
|
-
# Same entry as in previous example
|
342
|
-
|
343
|
-
# Given matched symbol is S[0, 5]
|
344
|
-
# And its reduce item is S => S + M . | 0
|
345
|
-
# Then add child nodes corresponding to the rhs symbols
|
346
|
-
# And make M[?, 5] the current symbol
|
347
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
348
|
-
expected_text = <<-SNIPPET
|
349
|
-
P[0, 5]
|
350
|
-
+- S[0, 5]
|
351
|
-
+- S[0, ?]
|
352
|
-
+- +[?, ?]: '(nil)'
|
353
|
-
+- M[?, 5]
|
354
|
-
SNIPPET
|
355
|
-
root_text = builder.root.to_string(0)
|
356
|
-
expect(root_text).to eq(expected_text.chomp)
|
357
|
-
expected_entry = 'S => S + M . | 0'
|
358
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
359
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
360
|
-
expect(builder.current_node.to_string(0)).to eq('M[?, 5]')
|
361
|
-
|
362
|
-
# Second similar test
|
363
|
-
|
364
|
-
# Given matched symbol is M[?, 5]
|
365
|
-
# And its reduce item is M => M * T . | 2
|
366
|
-
# Then add child nodes corresponding to the rhs symbols
|
367
|
-
# And make T[?, 5] the current symbol
|
368
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
369
|
-
expected_text = <<-SNIPPET
|
370
|
-
P[0, 5]
|
371
|
-
+- S[0, 5]
|
372
|
-
+- S[0, ?]
|
373
|
-
+- +[?, ?]: '(nil)'
|
374
|
-
+- M[2, 5]
|
375
|
-
+- M[2, ?]
|
376
|
-
+- *[?, ?]: '(nil)'
|
377
|
-
+- T[?, 5]
|
378
|
-
SNIPPET
|
379
|
-
root_text = builder.root.to_string(0)
|
380
|
-
expect(root_text).to eq(expected_text.chomp)
|
381
|
-
expected_entry = 'M => M * T . | 2'
|
382
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
383
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
384
|
-
expect(builder.current_node.to_string(0)).to eq('T[?, 5]')
|
385
|
-
end
|
386
|
-
|
387
|
-
|
388
300
|
|
389
|
-
it 'should
|
390
|
-
|
391
|
-
(
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
+- *[?, ?]: '(nil)'
|
408
|
-
+- T[4, 5]
|
409
|
-
+- integer[4, 5]: '(nil)'
|
410
|
-
SNIPPET
|
411
|
-
root_text = builder.root.to_string(0)
|
412
|
-
expect(root_text).to eq(expected_text.chomp)
|
413
|
-
expected_entry = 'T => integer . | 4'
|
414
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
415
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
416
|
-
integer_repr = "integer[4, 5]: '(nil)'"
|
417
|
-
expect(builder.current_node.to_string(0)).to eq(integer_repr)
|
418
|
-
|
419
|
-
# Given current tree symbol is integer[4, 5]: '(nil)'
|
420
|
-
# And its previous item is T => . integer | 4
|
421
|
-
# Then attach the token to the terminal node
|
422
|
-
# And decrement the entry index by one
|
423
|
-
# Make *[?, ?]: '(nil)' the current symbol
|
424
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
425
|
-
expected_text = <<-SNIPPET
|
426
|
-
P[0, 5]
|
427
|
-
+- S[0, 5]
|
428
|
-
+- S[0, ?]
|
429
|
-
+- +[?, ?]: '(nil)'
|
430
|
-
+- M[2, 5]
|
431
|
-
+- M[2, ?]
|
432
|
-
+- *[?, ?]: '(nil)'
|
433
|
-
+- T[4, 5]
|
434
|
-
+- integer[4, 5]: '4'
|
435
|
-
SNIPPET
|
436
|
-
root_text = builder.root.to_string(0)
|
437
|
-
expect(root_text).to eq(expected_text.chomp)
|
438
|
-
expected_entry = 'T => . integer | 4'
|
439
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
440
|
-
expect(entry_tracker.entry_set_index).to eq(4)
|
441
|
-
next_symbol = "*[?, ?]: '(nil)'"
|
442
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
301
|
+
it 'should give a text representation of itself' do
|
302
|
+
repr = subject.to_s
|
303
|
+
expect(repr).to match(/^success\? true/)
|
304
|
+
|
305
|
+
# Let's test the last chart state only
|
306
|
+
expectation = <<REPR
|
307
|
+
State[5]
|
308
|
+
T => integer . | 4
|
309
|
+
T. | 4
|
310
|
+
M => M * T . | 2
|
311
|
+
M. | 2
|
312
|
+
S => S + M . | 0
|
313
|
+
M => M . * T | 2
|
314
|
+
S. | 0
|
315
|
+
P => S . | 0
|
316
|
+
S => S . + M | 0
|
317
|
+
P. | 0
|
318
|
+
REPR
|
443
319
|
end
|
320
|
+
end # context
|
444
321
|
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
450
|
-
end
|
451
|
-
|
452
|
-
# Given current tree symbol is *[?, ?]: '(nil)'
|
453
|
-
# And current dotted item is T => . integer | 4
|
454
|
-
# When one retrieves the parse entry expecting the T
|
455
|
-
# Then new parse entry is changed to: M => M * . T | 2
|
456
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
457
|
-
|
458
|
-
expected_text = <<-SNIPPET
|
459
|
-
P[0, 5]
|
460
|
-
+- S[0, 5]
|
461
|
-
+- S[0, ?]
|
462
|
-
+- +[?, ?]: '(nil)'
|
463
|
-
+- M[2, 5]
|
464
|
-
+- M[2, ?]
|
465
|
-
+- *[?, ?]: '(nil)'
|
466
|
-
+- T[4, 5]
|
467
|
-
+- integer[4, 5]: '4'
|
468
|
-
SNIPPET
|
469
|
-
root_text = builder.root.to_string(0)
|
470
|
-
expect(root_text).to eq(expected_text.chomp)
|
471
|
-
expected_entry = 'M => M * . T | 2'
|
472
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
473
|
-
expect(entry_tracker.entry_set_index).to eq(4)
|
474
|
-
next_symbol = "*[?, ?]: '(nil)'"
|
475
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
476
|
-
|
477
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
478
|
-
next_symbol = 'M[2, ?]'
|
479
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
322
|
+
context 'Parse forest building:' do
|
323
|
+
let(:b_expr_grammar) do
|
324
|
+
builder = grammar_expr_builder
|
325
|
+
builder.grammar
|
480
326
|
end
|
481
327
|
|
482
|
-
|
483
|
-
|
484
|
-
is_done = false
|
485
|
-
(entry_tracker, builder) = prepare_parse_tree(subject)
|
486
|
-
16.times do
|
487
|
-
is_done = subject.insert_matched_symbol(entry_tracker, builder)
|
488
|
-
end
|
489
|
-
|
490
|
-
expected_text = <<-SNIPPET
|
491
|
-
P[0, 5]
|
492
|
-
+- S[0, 5]
|
493
|
-
+- S[0, 1]
|
494
|
-
+- M[0, 1]
|
495
|
-
+- T[0, 1]
|
496
|
-
+- integer[0, 1]: '2'
|
497
|
-
+- +[1, 2]: '+'
|
498
|
-
+- M[2, 5]
|
499
|
-
+- M[2, 3]
|
500
|
-
+- T[2, 3]
|
501
|
-
+- integer[2, 3]: '3'
|
502
|
-
+- *[3, 4]: '*'
|
503
|
-
+- T[4, 5]
|
504
|
-
+- integer[4, 5]: '4'
|
505
|
-
SNIPPET
|
506
|
-
root_text = builder.root.to_string(0)
|
507
|
-
expect(root_text).to eq(expected_text.chomp)
|
508
|
-
|
509
|
-
expected_entry = 'T => . integer | 0'
|
510
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
511
|
-
expect(entry_tracker.entry_set_index).to eq(0)
|
512
|
-
expect(is_done).to eq(true)
|
328
|
+
def grm_symbol(aSymbolName)
|
329
|
+
b_expr_grammar.name2symbol[aSymbolName]
|
513
330
|
end
|
514
331
|
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
parser
|
519
|
-
instance = parser.parse(token_seq1)
|
520
|
-
ptree = instance.parse_tree
|
521
|
-
expect(ptree).to be_kind_of(PTree::ParseTree)
|
332
|
+
subject do
|
333
|
+
parser = GFGEarleyParser.new(b_expr_grammar)
|
334
|
+
tokens = expr_tokenizer('3 * 4')
|
335
|
+
parser.parse(tokens)
|
522
336
|
end
|
523
337
|
|
524
|
-
it 'should
|
525
|
-
|
526
|
-
tokens = expr_tokenizer('2 + 3 * 4', b_expr_grammar)
|
527
|
-
instance = parser.parse(tokens)
|
528
|
-
ptree = instance.parse_tree
|
529
|
-
expect(ptree).to be_kind_of(PTree::ParseTree)
|
530
|
-
|
531
|
-
# Expect parse tree:
|
532
|
-
expected_text = <<-SNIPPET
|
533
|
-
P[0, 5]
|
534
|
-
+- S[0, 5]
|
535
|
-
+- S[0, 1]
|
536
|
-
+- M[0, 1]
|
537
|
-
+- T[0, 1]
|
538
|
-
+- integer[0, 1]: '2'
|
539
|
-
+- +[1, 2]: '+'
|
540
|
-
+- M[2, 5]
|
541
|
-
+- M[2, 3]
|
542
|
-
+- T[2, 3]
|
543
|
-
+- integer[2, 3]: '3'
|
544
|
-
+- *[3, 4]: '*'
|
545
|
-
+- T[4, 5]
|
546
|
-
+- integer[4, 5]: '4'
|
547
|
-
SNIPPET
|
548
|
-
actual = ptree.root.to_string(0)
|
549
|
-
expect(actual).to eq(expected_text.chomp)
|
338
|
+
it 'should indicate whether a parse succeeded' do
|
339
|
+
expect(subject.success?).to be_truthy
|
550
340
|
end
|
551
|
-
=end
|
552
341
|
end # context
|
553
342
|
end # describe
|
554
343
|
end # module
|