rley 0.7.03 → 0.7.08
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +362 -62
- data/.travis.yml +6 -7
- data/CHANGELOG.md +20 -1
- data/LICENSE.txt +1 -1
- data/README.md +6 -7
- data/Rakefile +2 -0
- data/appveyor.yml +2 -4
- data/examples/NLP/benchmark_pico_en.rb +2 -0
- data/examples/NLP/engtagger.rb +193 -188
- data/examples/NLP/nano_eng/nano_en_demo.rb +2 -0
- data/examples/NLP/nano_eng/nano_grammar.rb +7 -5
- data/examples/NLP/pico_en_demo.rb +2 -0
- data/examples/data_formats/JSON/cli_options.rb +3 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +14 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +14 -21
- data/examples/data_formats/JSON/json_demo.rb +2 -0
- data/examples/data_formats/JSON/json_grammar.rb +4 -2
- data/examples/data_formats/JSON/json_lexer.rb +10 -8
- data/examples/data_formats/JSON/json_minifier.rb +3 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +15 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +25 -37
- data/examples/general/calc_iter1/calc_demo.rb +2 -0
- data/examples/general/calc_iter1/calc_grammar.rb +4 -2
- data/examples/general/calc_iter1/calc_lexer.rb +8 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +7 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +7 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +29 -43
- data/examples/general/calc_iter2/calc_demo.rb +2 -0
- data/examples/general/calc_iter2/calc_grammar.rb +5 -3
- data/examples/general/calc_iter2/calc_lexer.rb +13 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +28 -26
- data/examples/general/left.rb +4 -2
- data/examples/general/right.rb +4 -2
- data/lib/rley.rb +2 -0
- data/lib/rley/base/base_parser.rb +2 -0
- data/lib/rley/base/dotted_item.rb +38 -41
- data/lib/rley/base/grm_items_builder.rb +2 -0
- data/lib/rley/constants.rb +5 -3
- data/lib/rley/engine.rb +22 -24
- data/lib/rley/formatter/asciitree.rb +6 -4
- data/lib/rley/formatter/base_formatter.rb +2 -0
- data/lib/rley/formatter/bracket_notation.rb +3 -8
- data/lib/rley/formatter/debug.rb +8 -6
- data/lib/rley/formatter/json.rb +4 -2
- data/lib/rley/gfg/call_edge.rb +3 -1
- data/lib/rley/gfg/edge.rb +7 -5
- data/lib/rley/gfg/end_vertex.rb +4 -6
- data/lib/rley/gfg/epsilon_edge.rb +3 -5
- data/lib/rley/gfg/grm_flow_graph.rb +31 -25
- data/lib/rley/gfg/item_vertex.rb +12 -22
- data/lib/rley/gfg/non_terminal_vertex.rb +6 -4
- data/lib/rley/gfg/return_edge.rb +2 -0
- data/lib/rley/gfg/scan_edge.rb +3 -1
- data/lib/rley/gfg/shortcut_edge.rb +4 -2
- data/lib/rley/gfg/start_vertex.rb +6 -8
- data/lib/rley/gfg/vertex.rb +47 -41
- data/lib/rley/lexical/token.rb +3 -1
- data/lib/rley/lexical/token_range.rb +8 -6
- data/lib/rley/parse_forest_visitor.rb +7 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +11 -11
- data/lib/rley/parse_rep/cst_builder.rb +7 -4
- data/lib/rley/parse_rep/parse_forest_builder.rb +36 -25
- data/lib/rley/parse_rep/parse_forest_factory.rb +5 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +18 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +15 -15
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -25
- data/lib/rley/parse_tree_visitor.rb +3 -1
- data/lib/rley/parser/error_reason.rb +9 -8
- data/lib/rley/parser/gfg_chart.rb +54 -22
- data/lib/rley/parser/gfg_earley_parser.rb +3 -1
- data/lib/rley/parser/gfg_parsing.rb +51 -31
- data/lib/rley/parser/parse_entry.rb +29 -33
- data/lib/rley/parser/parse_entry_set.rb +32 -27
- data/lib/rley/parser/parse_entry_tracker.rb +6 -4
- data/lib/rley/parser/parse_state.rb +18 -21
- data/lib/rley/parser/parse_state_tracker.rb +6 -4
- data/lib/rley/parser/parse_tracer.rb +15 -13
- data/lib/rley/parser/parse_walker_factory.rb +28 -29
- data/lib/rley/parser/state_set.rb +11 -10
- data/lib/rley/ptree/non_terminal_node.rb +10 -6
- data/lib/rley/ptree/parse_tree.rb +6 -4
- data/lib/rley/ptree/parse_tree_node.rb +7 -5
- data/lib/rley/ptree/terminal_node.rb +9 -7
- data/lib/rley/rley_error.rb +12 -10
- data/lib/rley/sppf/alternative_node.rb +8 -6
- data/lib/rley/sppf/composite_node.rb +9 -7
- data/lib/rley/sppf/epsilon_node.rb +5 -3
- data/lib/rley/sppf/leaf_node.rb +5 -3
- data/lib/rley/sppf/non_terminal_node.rb +2 -0
- data/lib/rley/sppf/parse_forest.rb +19 -17
- data/lib/rley/sppf/sppf_node.rb +9 -8
- data/lib/rley/sppf/token_node.rb +5 -3
- data/lib/rley/syntax/grammar.rb +7 -5
- data/lib/rley/syntax/grammar_builder.rb +11 -9
- data/lib/rley/syntax/grm_symbol.rb +8 -6
- data/lib/rley/syntax/literal.rb +2 -0
- data/lib/rley/syntax/non_terminal.rb +11 -15
- data/lib/rley/syntax/production.rb +13 -11
- data/lib/rley/syntax/symbol_seq.rb +10 -10
- data/lib/rley/syntax/terminal.rb +6 -5
- data/lib/rley/syntax/verbatim_symbol.rb +5 -3
- data/lib/support/base_tokenizer.rb +23 -20
- data/spec/rley/base/dotted_item_spec.rb +4 -2
- data/spec/rley/base/grm_items_builder_spec.rb +2 -0
- data/spec/rley/engine_spec.rb +47 -9
- data/spec/rley/formatter/asciitree_spec.rb +11 -9
- data/spec/rley/formatter/bracket_notation_spec.rb +16 -14
- data/spec/rley/formatter/debug_spec.rb +4 -2
- data/spec/rley/formatter/json_spec.rb +5 -3
- data/spec/rley/gfg/call_edge_spec.rb +2 -0
- data/spec/rley/gfg/edge_spec.rb +2 -0
- data/spec/rley/gfg/end_vertex_spec.rb +7 -5
- data/spec/rley/gfg/epsilon_edge_spec.rb +2 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -0
- data/spec/rley/gfg/item_vertex_spec.rb +12 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +5 -3
- data/spec/rley/gfg/return_edge_spec.rb +2 -0
- data/spec/rley/gfg/scan_edge_spec.rb +2 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +3 -1
- data/spec/rley/gfg/start_vertex_spec.rb +7 -5
- data/spec/rley/gfg/vertex_spec.rb +5 -3
- data/spec/rley/lexical/token_range_spec.rb +18 -16
- data/spec/rley/lexical/token_spec.rb +4 -2
- data/spec/rley/parse_forest_visitor_spec.rb +167 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +46 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +8 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +7 -5
- data/spec/rley/parse_rep/groucho_spec.rb +25 -25
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +28 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +8 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +4 -2
- data/spec/rley/parse_tree_visitor_spec.rb +12 -8
- data/spec/rley/parser/error_reason_spec.rb +8 -6
- data/spec/rley/parser/gfg_chart_spec.rb +17 -4
- data/spec/rley/parser/gfg_earley_parser_spec.rb +16 -11
- data/spec/rley/parser/gfg_parsing_spec.rb +41 -252
- data/spec/rley/parser/parse_entry_set_spec.rb +2 -0
- data/spec/rley/parser/parse_entry_spec.rb +21 -19
- data/spec/rley/parser/parse_state_spec.rb +7 -5
- data/spec/rley/parser/parse_tracer_spec.rb +16 -14
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -8
- data/spec/rley/parser/state_set_spec.rb +24 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +7 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +6 -4
- data/spec/rley/ptree/parse_tree_spec.rb +2 -0
- data/spec/rley/ptree/terminal_node_spec.rb +8 -6
- data/spec/rley/sppf/alternative_node_spec.rb +8 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +5 -3
- data/spec/rley/sppf/token_node_spec.rb +6 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +5 -4
- data/spec/rley/support/expectation_helper.rb +2 -0
- data/spec/rley/support/grammar_abc_helper.rb +4 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +6 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +6 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +6 -5
- data/spec/rley/support/grammar_helper.rb +2 -0
- data/spec/rley/support/grammar_l0_helper.rb +15 -16
- data/spec/rley/support/grammar_pb_helper.rb +8 -5
- data/spec/rley/support/grammar_sppf_helper.rb +3 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +7 -5
- data/spec/rley/syntax/grammar_spec.rb +8 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +3 -1
- data/spec/rley/syntax/literal_spec.rb +2 -0
- data/spec/rley/syntax/non_terminal_spec.rb +10 -8
- data/spec/rley/syntax/production_spec.rb +15 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +4 -2
- data/spec/rley/syntax/terminal_spec.rb +7 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +3 -1
- data/spec/spec_helper.rb +2 -12
- data/spec/support/base_tokenizer_spec.rb +9 -2
- metadata +21 -63
- data/.simplecov +0 -7
- data/Gemfile +0 -8
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
@@ -23,12 +25,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
23
25
|
builder = Syntax::GrammarBuilder.new do
|
24
26
|
add_terminals('a', 'b')
|
25
27
|
rule 'Phi' => 'S'
|
26
|
-
rule 'S' =>
|
27
|
-
rule 'S' =>
|
28
|
+
rule 'S' => 'A T'
|
29
|
+
rule 'S' => 'a T'
|
28
30
|
rule 'A' => 'a'
|
29
|
-
rule 'A' =>
|
31
|
+
rule 'A' => 'B A'
|
30
32
|
rule 'B' => []
|
31
|
-
rule 'T' =>
|
33
|
+
rule 'T' => 'b b b'
|
32
34
|
end
|
33
35
|
builder.grammar
|
34
36
|
end
|
@@ -47,11 +49,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
47
49
|
end
|
48
50
|
|
49
51
|
# Emit a text representation of the current path.
|
50
|
-
def path_to_s
|
52
|
+
def path_to_s
|
51
53
|
text_parts = subject.curr_path.map do |path_element|
|
52
54
|
path_element.to_string(0)
|
53
55
|
end
|
54
|
-
|
56
|
+
text_parts.join('/')
|
55
57
|
end
|
56
58
|
|
57
59
|
context 'Initialization:' do
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
@@ -36,11 +38,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
36
38
|
end
|
37
39
|
|
38
40
|
# Emit a text representation of the current path.
|
39
|
-
def path_to_s
|
41
|
+
def path_to_s
|
40
42
|
text_parts = subject.curr_path.map do |path_element|
|
41
43
|
path_element.to_string(0)
|
42
44
|
end
|
43
|
-
|
45
|
+
text_parts.join('/')
|
44
46
|
end
|
45
47
|
|
46
48
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../spec_helper'
|
2
4
|
|
3
5
|
require_relative '../../lib/rley/lexical/token'
|
@@ -9,7 +11,7 @@ require_relative './support/grammar_sppf_helper'
|
|
9
11
|
require_relative '../../lib/rley/parse_tree_visitor'
|
10
12
|
|
11
13
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
12
|
-
describe ParseTreeVisitor do
|
14
|
+
describe ParseTreeVisitor do
|
13
15
|
let(:grammar_abc) do
|
14
16
|
sandbox = Object.new
|
15
17
|
sandbox.extend(GrammarABCHelper)
|
@@ -43,7 +45,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
43
45
|
# Capital letters represent non-terminal nodes
|
44
46
|
let(:grm_abc_ptree1) do
|
45
47
|
engine = Rley::Engine.new
|
46
|
-
engine.use_grammar(grammar_abc)
|
48
|
+
engine.use_grammar(grammar_abc)
|
47
49
|
parse_result = engine.parse(grm_abc_tokens1)
|
48
50
|
ptree = engine.convert(parse_result)
|
49
51
|
ptree
|
@@ -159,6 +161,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
159
161
|
subject.end_visit_ptree(grm_abc_ptree1)
|
160
162
|
end
|
161
163
|
|
164
|
+
# rubocop: disable Naming/VariableNumber
|
162
165
|
it 'should begin the visit when requested' do
|
163
166
|
# Reminder: parse tree structure is
|
164
167
|
# S[0,5]
|
@@ -207,11 +210,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
207
210
|
expectations.each do |(msg, args)|
|
208
211
|
expect(listener1).to receive(msg).with(*args).ordered
|
209
212
|
end
|
210
|
-
|
213
|
+
|
211
214
|
# Here we go...
|
212
215
|
subject.start
|
213
216
|
end
|
214
|
-
|
217
|
+
|
215
218
|
it 'should also visit in pre-order' do
|
216
219
|
# Reminder: parse tree structure is
|
217
220
|
# S[0,5]
|
@@ -226,7 +229,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
226
229
|
root = grm_abc_ptree1.root
|
227
230
|
# Here we defeat encapsulation for the good cause
|
228
231
|
subject.instance_variable_set(:@traversal, :pre_order)
|
229
|
-
|
232
|
+
|
230
233
|
children = root.subnodes
|
231
234
|
big_a_1 = children[0]
|
232
235
|
big_a_1_children = big_a_1.subnodes
|
@@ -237,7 +240,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
237
240
|
expectations = [
|
238
241
|
[:before_ptree, [grm_abc_ptree1]]
|
239
242
|
# TODO: fix this test
|
240
|
-
# [:before_subnodes, [root, children]],
|
243
|
+
# [:before_subnodes, [root, children]],
|
241
244
|
# [:before_non_terminal, [root]],
|
242
245
|
|
243
246
|
# [:before_non_terminal, [big_a_1]],
|
@@ -265,10 +268,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
265
268
|
expectations.each do |(msg, args)|
|
266
269
|
expect(listener1).to receive(msg).with(*args).ordered
|
267
270
|
end
|
268
|
-
|
271
|
+
|
269
272
|
# Here we go...
|
270
273
|
subject.start
|
271
|
-
end
|
274
|
+
end
|
275
|
+
# rubocop: enable Naming/VariableNumber
|
272
276
|
end # context
|
273
277
|
end # describe
|
274
278
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require_relative '../../../lib/rley/lexical/token'
|
3
5
|
|
@@ -44,8 +46,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
44
46
|
|
45
47
|
context 'Initialization:' do
|
46
48
|
it 'should be created with arguments' do
|
47
|
-
expect do
|
48
|
-
ExpectationNotMet.new(3, err_token, terminals)
|
49
|
+
expect do
|
50
|
+
ExpectationNotMet.new(3, err_token, terminals)
|
49
51
|
end.not_to raise_error
|
50
52
|
end
|
51
53
|
|
@@ -73,8 +75,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
73
75
|
|
74
76
|
context 'Initialization:' do
|
75
77
|
it 'should be created with arguments' do
|
76
|
-
expect do
|
77
|
-
UnexpectedToken.new(3, err_token, terminals)
|
78
|
+
expect do
|
79
|
+
UnexpectedToken.new(3, err_token, terminals)
|
78
80
|
end.not_to raise_error
|
79
81
|
end
|
80
82
|
end # context
|
@@ -105,8 +107,8 @@ MESSAGE_END
|
|
105
107
|
|
106
108
|
context 'Initialization:' do
|
107
109
|
it 'should be created with arguments' do
|
108
|
-
expect do
|
109
|
-
PrematureInputEnd.new(3, err_token, terminals)
|
110
|
+
expect do
|
111
|
+
PrematureInputEnd.new(3, err_token, terminals)
|
110
112
|
end.not_to raise_error
|
111
113
|
end
|
112
114
|
end # context
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
|
@@ -75,13 +77,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
75
77
|
|
76
78
|
context 'Provided services:' do
|
77
79
|
it 'should accept the pushing of a parse entry in existing set' do
|
78
|
-
expect(subject.sets[0].entries.size).to eq(1)
|
80
|
+
expect(subject.sets[0].entries.size).to eq(1)
|
79
81
|
subject.push_entry(second_vertex, 0, 0, :scan_rule)
|
80
82
|
expect(subject.sets[0].entries.size).to eq(2)
|
81
83
|
end
|
82
|
-
|
84
|
+
|
83
85
|
it 'should accept the pushing of a parse entry in new set' do
|
84
|
-
expect(subject.sets[0].entries.size).to eq(1)
|
86
|
+
expect(subject.sets[0].entries.size).to eq(1)
|
85
87
|
subject.push_entry(second_vertex, 0, 1, :scan_rule)
|
86
88
|
expect(subject.sets[0].entries.size).to eq(1)
|
87
89
|
expect(subject.sets.size).to eq(2)
|
@@ -90,7 +92,18 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
90
92
|
|
91
93
|
it 'should retrieve an existing set at given position' do
|
92
94
|
expect(subject[0]).to eq(subject.sets[0])
|
93
|
-
end
|
95
|
+
end
|
96
|
+
|
97
|
+
it 'should a user-friendly text representation of itself' do
|
98
|
+
subject.push_entry(second_vertex, 0, 1, :scan_rule)
|
99
|
+
representation = <<REPR
|
100
|
+
State[0]
|
101
|
+
.S | 0
|
102
|
+
State[1]
|
103
|
+
S => . A | 0
|
104
|
+
REPR
|
105
|
+
expect(subject.to_s).to eq(representation)
|
106
|
+
end
|
94
107
|
end # context
|
95
108
|
end # describe
|
96
109
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
require_relative '../../../lib/rley/syntax/verbatim_symbol'
|
@@ -53,7 +55,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
53
55
|
let(:plus) { Syntax::VerbatimSymbol.new('+') }
|
54
56
|
let(:star) { Syntax::VerbatimSymbol.new('*') }
|
55
57
|
let(:integer) do
|
56
|
-
integer_pattern = /[-+]?[0-9]+/
|
58
|
+
integer_pattern = /[-+]?[0-9]+/ # Decimal notation
|
57
59
|
Syntax::Literal.new('integer', integer_pattern)
|
58
60
|
end
|
59
61
|
let(:prod_P) { Syntax::Production.new(nt_P, [nt_S]) }
|
@@ -69,7 +71,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
69
71
|
|
70
72
|
# Helper method that mimicks the output of a tokenizer
|
71
73
|
# for the language specified by grammar_expr
|
72
|
-
def grm2_tokens
|
74
|
+
def grm2_tokens
|
73
75
|
input_sequence = [
|
74
76
|
{ '2' => 'integer' },
|
75
77
|
'+',
|
@@ -77,7 +79,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
77
79
|
'*',
|
78
80
|
{ '4' => 'integer' }
|
79
81
|
]
|
80
|
-
|
82
|
+
build_token_sequence(input_sequence, grammar_expr)
|
81
83
|
end
|
82
84
|
|
83
85
|
# Default instantiation rule
|
@@ -102,6 +104,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
102
104
|
end # context
|
103
105
|
|
104
106
|
context 'Parsing: ' do
|
107
|
+
# rubocop: disable Naming/VariableNumber
|
105
108
|
it 'should parse a valid simple input' do
|
106
109
|
parse_result = subject.parse(grm1_tokens)
|
107
110
|
expect(parse_result.success?).to eq(true)
|
@@ -149,7 +152,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
149
152
|
expected = [
|
150
153
|
'A => b . | 2', # scan 'b'
|
151
154
|
'A. | 2', # exit rule
|
152
|
-
'A => a A . c | 1'
|
155
|
+
'A => a A . c | 1' # end rule
|
153
156
|
]
|
154
157
|
entry_set_3 = parse_result.chart[3]
|
155
158
|
expect(entry_set_3.entries.size).to eq(3)
|
@@ -181,6 +184,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
181
184
|
expect(entry_set_5.entries.size).to eq(4)
|
182
185
|
compare_entry_texts(entry_set_5, expected)
|
183
186
|
end
|
187
|
+
# rubocop: enable Naming/VariableNumber
|
184
188
|
|
185
189
|
it 'should parse a valid simple expression' do
|
186
190
|
instance = GFGEarleyParser.new(grammar_expr)
|
@@ -284,10 +288,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
284
288
|
# A => ;
|
285
289
|
t_x = Syntax::VerbatimSymbol.new('x')
|
286
290
|
|
287
|
-
builder = Syntax::GrammarBuilder.new
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
+
builder = Syntax::GrammarBuilder.new do
|
292
|
+
add_terminals(t_x)
|
293
|
+
rule 'Ss' => 'A A x'
|
294
|
+
rule 'A' => []
|
295
|
+
end
|
291
296
|
pos = Lexical::Position.new(1, 1)
|
292
297
|
tokens = [Lexical::Token.new('x', t_x, pos)]
|
293
298
|
|
@@ -406,7 +411,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
406
411
|
'P => S . | 0', # end rule
|
407
412
|
"S => S . '+' S | 0", # end rule
|
408
413
|
"S => S . '*' S | 0", # end rule
|
409
|
-
'P. | 0'
|
414
|
+
'P. | 0' # exit rule
|
410
415
|
]
|
411
416
|
compare_entry_texts(parse_result.chart[3], expected)
|
412
417
|
|
@@ -549,7 +554,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
549
554
|
'E => E . + E | 2', # end rule
|
550
555
|
'S => E . | 0', # end rule
|
551
556
|
'E => E . + E | 0', # end rule
|
552
|
-
'S. | 0'
|
557
|
+
'S. | 0' # exit rule
|
553
558
|
]
|
554
559
|
compare_entry_texts(parse_result.chart[5], expected)
|
555
560
|
end
|
@@ -618,7 +623,7 @@ MSG
|
|
618
623
|
]
|
619
624
|
compare_entry_texts(parse_result.chart[2], expected)
|
620
625
|
|
621
|
-
err_msg = "Premature end of input after '+' at position line 1, "
|
626
|
+
err_msg = +"Premature end of input after '+' at position line 1, "
|
622
627
|
err_msg << 'column 3'
|
623
628
|
err_msg << "\nExpected one of: ['int', '(']."
|
624
629
|
expect(parse_result.failure_reason.message).to eq(err_msg)
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
require 'stringio'
|
3
5
|
|
@@ -25,7 +27,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
25
27
|
include GrammarABCHelper # Mix-in module with builder for grammar abc
|
26
28
|
include GrammarBExprHelper # Mix-in with builder for simple expressions
|
27
29
|
include GrammarHelper # Mix-in with method for creating token sequence
|
28
|
-
|
30
|
+
|
29
31
|
# Helper method. Create an array of dotted items
|
30
32
|
# from the given grammar
|
31
33
|
def build_items_for_grammar(aGrammar)
|
@@ -93,14 +95,14 @@ SNIPPET
|
|
93
95
|
|
94
96
|
context 'Parsing:' do
|
95
97
|
# Utility method to fill the first entry set...
|
96
|
-
def fill_first_set
|
98
|
+
def fill_first_set
|
97
99
|
subject.start_rule(subject.initial_entry, 0)
|
98
100
|
subject.call_rule(subject.chart[0].last, 0)
|
99
101
|
subject.start_rule(subject.chart[0].last, 0)
|
100
102
|
end
|
101
103
|
|
102
104
|
# Utility method to initialize the second entry set...
|
103
|
-
def seed_second_set
|
105
|
+
def seed_second_set
|
104
106
|
# Cheating: we change the tokens to scan...
|
105
107
|
# Seeding second entry set...
|
106
108
|
subject.scan_rule(0, grm1_token_b[0])
|
@@ -164,7 +166,7 @@ SNIPPET
|
|
164
166
|
expect(subject.chart[0].size).to eq(3)
|
165
167
|
# Last entry is: (.A, 0)
|
166
168
|
dot_A_entry = subject.chart[0].last
|
167
|
-
|
169
|
+
|
168
170
|
subject.start_rule(dot_A_entry, 0)
|
169
171
|
|
170
172
|
# Expectations: two entries:
|
@@ -194,7 +196,7 @@ SNIPPET
|
|
194
196
|
expect(last_entry.vertex.label).to eq('A => a . A c')
|
195
197
|
expect(last_entry.origin).to eq(0)
|
196
198
|
antecedence = subject.antecedence
|
197
|
-
expect(antecedence.fetch(last_entry)).to eq([fourth_entry])
|
199
|
+
expect(antecedence.fetch(last_entry)).to eq([fourth_entry])
|
198
200
|
end
|
199
201
|
|
200
202
|
it 'should apply the exit rule correctly' do
|
@@ -219,7 +221,7 @@ SNIPPET
|
|
219
221
|
exit_entry = subject.chart[1].last
|
220
222
|
expect(exit_entry.vertex.label).to eq('A.')
|
221
223
|
expect(exit_entry.origin).to eq(0)
|
222
|
-
expect(subject.antecedence.fetch(exit_entry)).to eq([last_entry])
|
224
|
+
expect(subject.antecedence.fetch(exit_entry)).to eq([last_entry])
|
223
225
|
end
|
224
226
|
|
225
227
|
it 'should apply the end rule correctly' do
|
@@ -248,10 +250,8 @@ SNIPPET
|
|
248
250
|
expect(end_entry.origin).to eq(0)
|
249
251
|
expect(subject.antecedence.fetch(end_entry)).to eq([exit_entry])
|
250
252
|
end
|
251
|
-
=begin
|
252
|
-
|
253
|
-
|
254
253
|
|
254
|
+
=begin
|
255
255
|
it 'should retrieve the parse states that expect a given terminal' do
|
256
256
|
item1 = DottedItem.new(prod_A1, 2)
|
257
257
|
item2 = DottedItem.new(prod_A1, 1)
|
@@ -282,18 +282,7 @@ SNIPPET
|
|
282
282
|
=end
|
283
283
|
end # context
|
284
284
|
|
285
|
-
context '
|
286
|
-
let(:sample_grammar1) do
|
287
|
-
builder = grammar_abc_builder
|
288
|
-
builder.grammar
|
289
|
-
end
|
290
|
-
|
291
|
-
let(:token_seq1) do
|
292
|
-
%w[a a b c c].map do |letter|
|
293
|
-
Lexical::Token.new(letter, sample_grammar1.name2symbol[letter])
|
294
|
-
end
|
295
|
-
end
|
296
|
-
|
285
|
+
context 'Provided services:' do
|
297
286
|
let(:b_expr_grammar) do
|
298
287
|
builder = grammar_expr_builder
|
299
288
|
builder.grammar
|
@@ -308,247 +297,47 @@ SNIPPET
|
|
308
297
|
tokens = expr_tokenizer('2 + 3 * 4')
|
309
298
|
parser.parse(tokens)
|
310
299
|
end
|
311
|
-
|
312
|
-
it 'should indicate whether a parse succeeded' do
|
313
|
-
expect(subject.success?).to be_truthy
|
314
|
-
end
|
315
|
-
|
316
|
-
it 'should build a parse forest' do
|
317
|
-
expect { subject.parse_forest }.not_to raise_error if subject.success?
|
318
|
-
end
|
319
|
-
=begin
|
320
|
-
it 'should create the root of a parse forest' do
|
321
|
-
(entry_tracker, builder) = prepare_parse_forest(subject)
|
322
|
-
# The root node should correspond to the start symbol and
|
323
|
-
# its direct children should correspond to rhs of start production
|
324
|
-
expected_text = <<-SNIPPET
|
325
|
-
P[0, 5]
|
326
|
-
+- S[0, 5]
|
327
|
-
SNIPPET
|
328
|
-
root_text = builder.root.to_string(0)
|
329
|
-
expect(root_text).to eq(expected_text.chomp)
|
330
|
-
|
331
|
-
expect(entry_tracker.entry_set_index).to eq(subject.tokens.size)
|
332
|
-
expected_entry = 'P => S . | 0'
|
333
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
334
|
-
expect(builder.current_node.to_string(0)).to eq('S[0, 5]')
|
335
|
-
end
|
336
|
-
=end
|
337
|
-
=begin
|
338
|
-
it 'should use a reduce item for a matched non-terminal' do
|
339
|
-
# Setup
|
340
|
-
(entry_tracker, builder) = prepare_parse_tree(subject)
|
341
|
-
# Same entry as in previous example
|
342
|
-
|
343
|
-
# Given matched symbol is S[0, 5]
|
344
|
-
# And its reduce item is S => S + M . | 0
|
345
|
-
# Then add child nodes corresponding to the rhs symbols
|
346
|
-
# And make M[?, 5] the current symbol
|
347
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
348
|
-
expected_text = <<-SNIPPET
|
349
|
-
P[0, 5]
|
350
|
-
+- S[0, 5]
|
351
|
-
+- S[0, ?]
|
352
|
-
+- +[?, ?]: '(nil)'
|
353
|
-
+- M[?, 5]
|
354
|
-
SNIPPET
|
355
|
-
root_text = builder.root.to_string(0)
|
356
|
-
expect(root_text).to eq(expected_text.chomp)
|
357
|
-
expected_entry = 'S => S + M . | 0'
|
358
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
359
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
360
|
-
expect(builder.current_node.to_string(0)).to eq('M[?, 5]')
|
361
|
-
|
362
|
-
# Second similar test
|
363
|
-
|
364
|
-
# Given matched symbol is M[?, 5]
|
365
|
-
# And its reduce item is M => M * T . | 2
|
366
|
-
# Then add child nodes corresponding to the rhs symbols
|
367
|
-
# And make T[?, 5] the current symbol
|
368
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
369
|
-
expected_text = <<-SNIPPET
|
370
|
-
P[0, 5]
|
371
|
-
+- S[0, 5]
|
372
|
-
+- S[0, ?]
|
373
|
-
+- +[?, ?]: '(nil)'
|
374
|
-
+- M[2, 5]
|
375
|
-
+- M[2, ?]
|
376
|
-
+- *[?, ?]: '(nil)'
|
377
|
-
+- T[?, 5]
|
378
|
-
SNIPPET
|
379
|
-
root_text = builder.root.to_string(0)
|
380
|
-
expect(root_text).to eq(expected_text.chomp)
|
381
|
-
expected_entry = 'M => M * T . | 2'
|
382
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
383
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
384
|
-
expect(builder.current_node.to_string(0)).to eq('T[?, 5]')
|
385
|
-
end
|
386
|
-
|
387
|
-
|
388
300
|
|
389
|
-
it 'should
|
390
|
-
|
391
|
-
(
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
+- *[?, ?]: '(nil)'
|
408
|
-
+- T[4, 5]
|
409
|
-
+- integer[4, 5]: '(nil)'
|
410
|
-
SNIPPET
|
411
|
-
root_text = builder.root.to_string(0)
|
412
|
-
expect(root_text).to eq(expected_text.chomp)
|
413
|
-
expected_entry = 'T => integer . | 4'
|
414
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
415
|
-
expect(entry_tracker.entry_set_index).to eq(5)
|
416
|
-
integer_repr = "integer[4, 5]: '(nil)'"
|
417
|
-
expect(builder.current_node.to_string(0)).to eq(integer_repr)
|
418
|
-
|
419
|
-
# Given current tree symbol is integer[4, 5]: '(nil)'
|
420
|
-
# And its previous item is T => . integer | 4
|
421
|
-
# Then attach the token to the terminal node
|
422
|
-
# And decrement the entry index by one
|
423
|
-
# Make *[?, ?]: '(nil)' the current symbol
|
424
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
425
|
-
expected_text = <<-SNIPPET
|
426
|
-
P[0, 5]
|
427
|
-
+- S[0, 5]
|
428
|
-
+- S[0, ?]
|
429
|
-
+- +[?, ?]: '(nil)'
|
430
|
-
+- M[2, 5]
|
431
|
-
+- M[2, ?]
|
432
|
-
+- *[?, ?]: '(nil)'
|
433
|
-
+- T[4, 5]
|
434
|
-
+- integer[4, 5]: '4'
|
435
|
-
SNIPPET
|
436
|
-
root_text = builder.root.to_string(0)
|
437
|
-
expect(root_text).to eq(expected_text.chomp)
|
438
|
-
expected_entry = 'T => . integer | 4'
|
439
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
440
|
-
expect(entry_tracker.entry_set_index).to eq(4)
|
441
|
-
next_symbol = "*[?, ?]: '(nil)'"
|
442
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
301
|
+
it 'should give a text representation of itself' do
|
302
|
+
repr = subject.to_s
|
303
|
+
expect(repr).to match(/^success\? true/)
|
304
|
+
|
305
|
+
# Let's test the last chart state only
|
306
|
+
expectation = <<REPR
|
307
|
+
State[5]
|
308
|
+
T => integer . | 4
|
309
|
+
T. | 4
|
310
|
+
M => M * T . | 2
|
311
|
+
M. | 2
|
312
|
+
S => S + M . | 0
|
313
|
+
M => M . * T | 2
|
314
|
+
S. | 0
|
315
|
+
P => S . | 0
|
316
|
+
S => S . + M | 0
|
317
|
+
P. | 0
|
318
|
+
REPR
|
443
319
|
end
|
320
|
+
end # context
|
444
321
|
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
450
|
-
end
|
451
|
-
|
452
|
-
# Given current tree symbol is *[?, ?]: '(nil)'
|
453
|
-
# And current dotted item is T => . integer | 4
|
454
|
-
# When one retrieves the parse entry expecting the T
|
455
|
-
# Then new parse entry is changed to: M => M * . T | 2
|
456
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
457
|
-
|
458
|
-
expected_text = <<-SNIPPET
|
459
|
-
P[0, 5]
|
460
|
-
+- S[0, 5]
|
461
|
-
+- S[0, ?]
|
462
|
-
+- +[?, ?]: '(nil)'
|
463
|
-
+- M[2, 5]
|
464
|
-
+- M[2, ?]
|
465
|
-
+- *[?, ?]: '(nil)'
|
466
|
-
+- T[4, 5]
|
467
|
-
+- integer[4, 5]: '4'
|
468
|
-
SNIPPET
|
469
|
-
root_text = builder.root.to_string(0)
|
470
|
-
expect(root_text).to eq(expected_text.chomp)
|
471
|
-
expected_entry = 'M => M * . T | 2'
|
472
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
473
|
-
expect(entry_tracker.entry_set_index).to eq(4)
|
474
|
-
next_symbol = "*[?, ?]: '(nil)'"
|
475
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
476
|
-
|
477
|
-
subject.insert_matched_symbol(entry_tracker, builder)
|
478
|
-
next_symbol = 'M[2, ?]'
|
479
|
-
expect(builder.current_node.to_string(0)).to eq(next_symbol)
|
322
|
+
context 'Parse forest building:' do
|
323
|
+
let(:b_expr_grammar) do
|
324
|
+
builder = grammar_expr_builder
|
325
|
+
builder.grammar
|
480
326
|
end
|
481
327
|
|
482
|
-
|
483
|
-
|
484
|
-
is_done = false
|
485
|
-
(entry_tracker, builder) = prepare_parse_tree(subject)
|
486
|
-
16.times do
|
487
|
-
is_done = subject.insert_matched_symbol(entry_tracker, builder)
|
488
|
-
end
|
489
|
-
|
490
|
-
expected_text = <<-SNIPPET
|
491
|
-
P[0, 5]
|
492
|
-
+- S[0, 5]
|
493
|
-
+- S[0, 1]
|
494
|
-
+- M[0, 1]
|
495
|
-
+- T[0, 1]
|
496
|
-
+- integer[0, 1]: '2'
|
497
|
-
+- +[1, 2]: '+'
|
498
|
-
+- M[2, 5]
|
499
|
-
+- M[2, 3]
|
500
|
-
+- T[2, 3]
|
501
|
-
+- integer[2, 3]: '3'
|
502
|
-
+- *[3, 4]: '*'
|
503
|
-
+- T[4, 5]
|
504
|
-
+- integer[4, 5]: '4'
|
505
|
-
SNIPPET
|
506
|
-
root_text = builder.root.to_string(0)
|
507
|
-
expect(root_text).to eq(expected_text.chomp)
|
508
|
-
|
509
|
-
expected_entry = 'T => . integer | 0'
|
510
|
-
expect(entry_tracker.parse_entry.to_s).to eq(expected_entry)
|
511
|
-
expect(entry_tracker.entry_set_index).to eq(0)
|
512
|
-
expect(is_done).to eq(true)
|
328
|
+
def grm_symbol(aSymbolName)
|
329
|
+
b_expr_grammar.name2symbol[aSymbolName]
|
513
330
|
end
|
514
331
|
|
515
|
-
|
516
|
-
|
517
|
-
|
518
|
-
parser
|
519
|
-
instance = parser.parse(token_seq1)
|
520
|
-
ptree = instance.parse_tree
|
521
|
-
expect(ptree).to be_kind_of(PTree::ParseTree)
|
332
|
+
subject do
|
333
|
+
parser = GFGEarleyParser.new(b_expr_grammar)
|
334
|
+
tokens = expr_tokenizer('3 * 4')
|
335
|
+
parser.parse(tokens)
|
522
336
|
end
|
523
337
|
|
524
|
-
it 'should
|
525
|
-
|
526
|
-
tokens = expr_tokenizer('2 + 3 * 4', b_expr_grammar)
|
527
|
-
instance = parser.parse(tokens)
|
528
|
-
ptree = instance.parse_tree
|
529
|
-
expect(ptree).to be_kind_of(PTree::ParseTree)
|
530
|
-
|
531
|
-
# Expect parse tree:
|
532
|
-
expected_text = <<-SNIPPET
|
533
|
-
P[0, 5]
|
534
|
-
+- S[0, 5]
|
535
|
-
+- S[0, 1]
|
536
|
-
+- M[0, 1]
|
537
|
-
+- T[0, 1]
|
538
|
-
+- integer[0, 1]: '2'
|
539
|
-
+- +[1, 2]: '+'
|
540
|
-
+- M[2, 5]
|
541
|
-
+- M[2, 3]
|
542
|
-
+- T[2, 3]
|
543
|
-
+- integer[2, 3]: '3'
|
544
|
-
+- *[3, 4]: '*'
|
545
|
-
+- T[4, 5]
|
546
|
-
+- integer[4, 5]: '4'
|
547
|
-
SNIPPET
|
548
|
-
actual = ptree.root.to_string(0)
|
549
|
-
expect(actual).to eq(expected_text.chomp)
|
338
|
+
it 'should indicate whether a parse succeeded' do
|
339
|
+
expect(subject.success?).to be_truthy
|
550
340
|
end
|
551
|
-
=end
|
552
341
|
end # context
|
553
342
|
end # describe
|
554
343
|
end # module
|