rley 0.7.07 → 0.7.08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +348 -54
- data/LICENSE.txt +1 -1
- data/README.md +3 -2
- data/examples/NLP/engtagger.rb +193 -190
- data/examples/NLP/nano_eng/nano_grammar.rb +5 -5
- data/examples/data_formats/JSON/cli_options.rb +1 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +12 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +12 -21
- data/examples/data_formats/JSON/json_grammar.rb +2 -2
- data/examples/data_formats/JSON/json_lexer.rb +8 -8
- data/examples/data_formats/JSON/json_minifier.rb +1 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +13 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +23 -37
- data/examples/general/calc_iter1/calc_grammar.rb +2 -2
- data/examples/general/calc_iter1/calc_lexer.rb +6 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +5 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +5 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +27 -43
- data/examples/general/calc_iter2/calc_grammar.rb +3 -3
- data/examples/general/calc_iter2/calc_lexer.rb +11 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +26 -26
- data/examples/general/left.rb +2 -2
- data/examples/general/right.rb +2 -2
- data/lib/rley/base/dotted_item.rb +23 -31
- data/lib/rley/constants.rb +2 -2
- data/lib/rley/engine.rb +20 -23
- data/lib/rley/formatter/asciitree.rb +3 -3
- data/lib/rley/formatter/bracket_notation.rb +1 -8
- data/lib/rley/formatter/debug.rb +6 -6
- data/lib/rley/formatter/json.rb +2 -2
- data/lib/rley/gfg/call_edge.rb +1 -1
- data/lib/rley/gfg/edge.rb +5 -5
- data/lib/rley/gfg/end_vertex.rb +2 -6
- data/lib/rley/gfg/epsilon_edge.rb +1 -5
- data/lib/rley/gfg/grm_flow_graph.rb +27 -23
- data/lib/rley/gfg/item_vertex.rb +10 -10
- data/lib/rley/gfg/non_terminal_vertex.rb +4 -4
- data/lib/rley/gfg/scan_edge.rb +1 -1
- data/lib/rley/gfg/shortcut_edge.rb +2 -2
- data/lib/rley/gfg/start_vertex.rb +4 -8
- data/lib/rley/gfg/vertex.rb +43 -39
- data/lib/rley/lexical/token_range.rb +6 -6
- data/lib/rley/parse_forest_visitor.rb +5 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +9 -11
- data/lib/rley/parse_rep/cst_builder.rb +5 -6
- data/lib/rley/parse_rep/parse_forest_builder.rb +20 -18
- data/lib/rley/parse_rep/parse_forest_factory.rb +3 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +11 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +4 -4
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -27
- data/lib/rley/parse_tree_visitor.rb +1 -1
- data/lib/rley/parser/error_reason.rb +4 -5
- data/lib/rley/parser/gfg_chart.rb +20 -22
- data/lib/rley/parser/gfg_parsing.rb +16 -30
- data/lib/rley/parser/parse_entry.rb +25 -31
- data/lib/rley/parser/parse_entry_set.rb +18 -15
- data/lib/rley/parser/parse_entry_tracker.rb +4 -4
- data/lib/rley/parser/parse_state.rb +16 -21
- data/lib/rley/parser/parse_state_tracker.rb +4 -4
- data/lib/rley/parser/parse_tracer.rb +13 -13
- data/lib/rley/parser/parse_walker_factory.rb +23 -28
- data/lib/rley/parser/state_set.rb +9 -10
- data/lib/rley/ptree/non_terminal_node.rb +7 -5
- data/lib/rley/ptree/parse_tree.rb +3 -3
- data/lib/rley/ptree/parse_tree_node.rb +5 -5
- data/lib/rley/ptree/terminal_node.rb +7 -7
- data/lib/rley/rley_error.rb +12 -12
- data/lib/rley/sppf/alternative_node.rb +6 -6
- data/lib/rley/sppf/composite_node.rb +7 -7
- data/lib/rley/sppf/epsilon_node.rb +3 -3
- data/lib/rley/sppf/leaf_node.rb +3 -3
- data/lib/rley/sppf/parse_forest.rb +16 -16
- data/lib/rley/sppf/sppf_node.rb +7 -8
- data/lib/rley/sppf/token_node.rb +3 -3
- data/lib/rley/syntax/grammar.rb +5 -5
- data/lib/rley/syntax/grammar_builder.rb +9 -9
- data/lib/rley/syntax/grm_symbol.rb +6 -6
- data/lib/rley/syntax/non_terminal.rb +9 -15
- data/lib/rley/syntax/production.rb +10 -10
- data/lib/rley/syntax/symbol_seq.rb +7 -9
- data/lib/rley/syntax/terminal.rb +4 -5
- data/lib/rley/syntax/verbatim_symbol.rb +3 -3
- data/lib/support/base_tokenizer.rb +19 -18
- data/spec/rley/base/dotted_item_spec.rb +2 -2
- data/spec/rley/engine_spec.rb +17 -15
- data/spec/rley/formatter/asciitree_spec.rb +7 -7
- data/spec/rley/formatter/bracket_notation_spec.rb +13 -13
- data/spec/rley/formatter/json_spec.rb +1 -1
- data/spec/rley/gfg/end_vertex_spec.rb +5 -5
- data/spec/rley/gfg/item_vertex_spec.rb +10 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +3 -3
- data/spec/rley/gfg/shortcut_edge_spec.rb +1 -1
- data/spec/rley/gfg/start_vertex_spec.rb +5 -5
- data/spec/rley/gfg/vertex_spec.rb +3 -3
- data/spec/rley/lexical/token_range_spec.rb +16 -16
- data/spec/rley/lexical/token_spec.rb +2 -2
- data/spec/rley/parse_forest_visitor_spec.rb +165 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +44 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +6 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +5 -5
- data/spec/rley/parse_rep/groucho_spec.rb +21 -21
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +26 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +6 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +2 -2
- data/spec/rley/parse_tree_visitor_spec.rb +10 -8
- data/spec/rley/parser/error_reason_spec.rb +6 -6
- data/spec/rley/parser/gfg_earley_parser_spec.rb +4 -2
- data/spec/rley/parser/gfg_parsing_spec.rb +4 -8
- data/spec/rley/parser/parse_entry_spec.rb +19 -19
- data/spec/rley/parser/parse_state_spec.rb +5 -5
- data/spec/rley/parser/parse_walker_factory_spec.rb +1 -1
- data/spec/rley/parser/state_set_spec.rb +22 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +5 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +4 -4
- data/spec/rley/ptree/terminal_node_spec.rb +6 -6
- data/spec/rley/sppf/alternative_node_spec.rb +6 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +3 -3
- data/spec/rley/sppf/token_node_spec.rb +4 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +3 -4
- data/spec/rley/support/grammar_abc_helper.rb +2 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +4 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +4 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +4 -5
- data/spec/rley/support/grammar_l0_helper.rb +10 -11
- data/spec/rley/support/grammar_pb_helper.rb +6 -5
- data/spec/rley/support/grammar_sppf_helper.rb +1 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +5 -5
- data/spec/rley/syntax/grammar_spec.rb +6 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +1 -1
- data/spec/rley/syntax/non_terminal_spec.rb +8 -8
- data/spec/rley/syntax/production_spec.rb +13 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +2 -2
- data/spec/rley/syntax/terminal_spec.rb +5 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -12
- data/spec/support/base_tokenizer_spec.rb +7 -2
- metadata +21 -62
- data/.simplecov +0 -8
@@ -33,7 +33,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
33
33
|
terminal = OpenStruct.new(name: aSymbolName)
|
34
34
|
return OpenStruct.new(lexeme: aLexeme, terminal: terminal)
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
it 'should accept the addition of subnodes' do
|
38
38
|
child1 = double('first_child')
|
39
39
|
child2 = double('second_child')
|
@@ -44,6 +44,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
44
44
|
expect(subject.subnodes).to eq([child3, child2, child1])
|
45
45
|
end
|
46
46
|
|
47
|
+
# rubocop: disable Naming/VariableNumber
|
47
48
|
it 'should provide a text representation of itself' do
|
48
49
|
# Case 1: no child
|
49
50
|
expected_text = 'VP[0, 3]'
|
@@ -61,9 +62,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
61
62
|
noun = build_token('bus', 'Noun')
|
62
63
|
child_3_1 = TerminalNode.new(noun, range(2, 3))
|
63
64
|
# We reverse the sequence of subnode addition
|
64
|
-
subject.add_subnode(child_1_2)
|
65
|
+
subject.add_subnode(child_1_2)
|
65
66
|
subject.add_subnode(child_1_1)
|
66
|
-
child_1_2.add_subnode(child_2_2)
|
67
|
+
child_1_2.add_subnode(child_2_2)
|
67
68
|
child_1_2.add_subnode(child_2_1)
|
68
69
|
child_2_2.add_subnode(child_3_1)
|
69
70
|
expected_text = <<-SNIPPET
|
@@ -76,6 +77,7 @@ VP[0, 3]
|
|
76
77
|
SNIPPET
|
77
78
|
expect(subject.to_string(0)).to eq(expected_text.chomp)
|
78
79
|
end
|
80
|
+
# rubocop: enable Naming/VariableNumber
|
79
81
|
end # context
|
80
82
|
end # describe
|
81
83
|
end # module
|
@@ -27,17 +27,17 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
27
27
|
expect(subject.range).to eq(sample_range)
|
28
28
|
end
|
29
29
|
end # context
|
30
|
-
|
30
|
+
|
31
31
|
context 'Initialization:' do
|
32
32
|
it 'should assign undefined range bounds' do
|
33
33
|
partial_range = { low: 0 } # High bound left undefined
|
34
34
|
instance = ParseTreeNode.new(sample_symbol, partial_range)
|
35
|
-
|
35
|
+
|
36
36
|
another = { low: 1, high: 4 } # High bound is specified
|
37
37
|
instance.range = another
|
38
38
|
expect(instance.range).to eq(low: 0, high: 4)
|
39
|
-
end
|
40
|
-
end # context
|
39
|
+
end
|
40
|
+
end # context
|
41
41
|
end # describe
|
42
42
|
end # module
|
43
43
|
end # module
|
@@ -10,23 +10,23 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
10
10
|
module PTree # Open this namespace to avoid module qualifier prefixes
|
11
11
|
describe TerminalNode do
|
12
12
|
let(:sample_symbol) { OpenStruct.new(name: 'Noun') }
|
13
|
-
let(:sample_token) do
|
14
|
-
OpenStruct.new(lexeme: 'world', terminal: sample_symbol)
|
13
|
+
let(:sample_token) do
|
14
|
+
OpenStruct.new(lexeme: 'world', terminal: sample_symbol)
|
15
15
|
end
|
16
16
|
let(:sample_range) { double('fake-range') }
|
17
|
-
|
17
|
+
|
18
18
|
subject { TerminalNode.new(sample_token, sample_range) }
|
19
|
-
|
19
|
+
|
20
20
|
context 'Initialization:' do
|
21
21
|
it 'should be bound to a token' do
|
22
22
|
expect(subject.token).to eq(sample_token)
|
23
23
|
end
|
24
24
|
end # context
|
25
|
-
|
25
|
+
|
26
26
|
context 'Provided services:' do
|
27
27
|
it 'should provide a text representation of itself' do
|
28
28
|
expected_text = "Noun[?, ?]: 'world'"
|
29
|
-
expect(subject.to_string(0)).to eq(expected_text)
|
29
|
+
expect(subject.to_string(0)).to eq(expected_text)
|
30
30
|
end
|
31
31
|
end # context
|
32
32
|
end # describe
|
@@ -19,7 +19,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
19
19
|
def range(low, high)
|
20
20
|
return Lexical::TokenRange.new(low: low, high: high)
|
21
21
|
end
|
22
|
-
|
22
|
+
|
23
23
|
let(:t_a) { Syntax::Terminal.new('A') }
|
24
24
|
let(:t_b) { Syntax::Terminal.new('B') }
|
25
25
|
let(:t_c) { Syntax::Terminal.new('C') }
|
@@ -29,7 +29,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
29
29
|
end
|
30
30
|
let(:sample_item) { Base::DottedItem.new(sample_prod, 3) }
|
31
31
|
let(:sample_vertex) { GFG::ItemVertex.new(sample_item) }
|
32
|
-
let(:sample_range) { range(0, 3) }
|
32
|
+
let(:sample_range) { range(0, 3) }
|
33
33
|
|
34
34
|
subject { AlternativeNode.new(sample_vertex, sample_range) }
|
35
35
|
|
@@ -37,9 +37,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
37
37
|
it 'should be created with a item vertex and a token range' do
|
38
38
|
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
39
39
|
.not_to raise_error
|
40
|
-
end
|
40
|
+
end
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
context 'Initialization:' do
|
44
44
|
it 'should know its token range' do
|
45
45
|
expect(subject.range).to eq(sample_range)
|
@@ -50,7 +50,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
50
50
|
expect(subject.subnodes).to be_empty
|
51
51
|
end
|
52
52
|
end # context
|
53
|
-
|
53
|
+
|
54
54
|
context 'Provided services:' do
|
55
55
|
it 'should accept the addition of subnodes' do
|
56
56
|
subnode1 = double('first_subnode')
|
@@ -63,7 +63,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
63
63
|
end
|
64
64
|
|
65
65
|
|
66
|
-
it 'should have a string representation' do
|
66
|
+
it 'should have a string representation' do
|
67
67
|
expect(subject.to_string(0)).to eq('Alt(sentence => A B C .)[0, 3]')
|
68
68
|
end
|
69
69
|
end # context
|
@@ -37,12 +37,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
37
37
|
it "shouldn't have children yet" do
|
38
38
|
expect(subject.subnodes).to be_empty
|
39
39
|
end
|
40
|
-
|
40
|
+
|
41
41
|
it 'should have :and refinement' do
|
42
42
|
expect(subject.refinement).to eq(:and)
|
43
43
|
end
|
44
44
|
end # context
|
45
|
-
|
45
|
+
|
46
46
|
context 'Provided services:' do
|
47
47
|
it 'should accept the addition of subnodes' do
|
48
48
|
subnode1 = double('first_subnode')
|
@@ -57,7 +57,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
57
57
|
it 'should have a string representation' do
|
58
58
|
expect(subject.to_string(0)).to eq('VP[0, 3]')
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
it 'should return a key value of itself' do
|
62
62
|
expect(subject.key).to eq('VP[0, 3]')
|
63
63
|
end
|
@@ -15,8 +15,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
15
15
|
describe TokenNode do
|
16
16
|
let(:sample_symbol) { Syntax::Terminal.new('Noun') }
|
17
17
|
let(:sample_position) { Lexical::Position.new(3, 4) }
|
18
|
-
let(:sample_token) do
|
19
|
-
Lexical::Token.new('language', sample_symbol, sample_position)
|
18
|
+
let(:sample_token) do
|
19
|
+
Lexical::Token.new('language', sample_symbol, sample_position)
|
20
20
|
end
|
21
21
|
let(:sample_rank) { 3 }
|
22
22
|
|
@@ -33,13 +33,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
33
33
|
expect(subject.range.high).to eq(sample_rank + 1)
|
34
34
|
end
|
35
35
|
end # context
|
36
|
-
|
36
|
+
|
37
37
|
context 'Provided services:' do
|
38
38
|
it 'should know its string representation' do
|
39
39
|
expect(subject.to_string(0)).to eq('Noun[3, 4]')
|
40
40
|
expect(subject.inspect).to eq('Noun[3, 4]')
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
it 'should return a key value of itself' do
|
44
44
|
expect(subject.key).to eq('Noun[3, 4]')
|
45
45
|
end
|
@@ -9,14 +9,13 @@ module AmbiguousGrammarHelper
|
|
9
9
|
# Factory method. Creates a grammar builder for a basic ambiguous
|
10
10
|
# expression grammar.
|
11
11
|
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
12
|
-
def grammar_builder
|
13
|
-
|
12
|
+
def grammar_builder
|
13
|
+
Rley::Syntax::GrammarBuilder.new do
|
14
14
|
add_terminals('+', 'id')
|
15
15
|
rule 'S' => 'E'
|
16
|
-
rule 'E' =>
|
16
|
+
rule 'E' => 'E + E'
|
17
17
|
rule 'E' => 'id'
|
18
18
|
end
|
19
|
-
builder
|
20
19
|
end
|
21
20
|
|
22
21
|
# Basic tokenizing method
|
@@ -6,15 +6,13 @@ require_relative '../../../lib/rley/syntax/grammar_builder'
|
|
6
6
|
module GrammarABCHelper
|
7
7
|
# Factory method. Creates a grammar builder for a simple grammar.
|
8
8
|
# (based on example in N. Wirth "Compiler Construction" book, p. 6)
|
9
|
-
def grammar_abc_builder
|
10
|
-
|
9
|
+
def grammar_abc_builder
|
10
|
+
Rley::Syntax::GrammarBuilder.new do
|
11
11
|
add_terminals('a', 'b', 'c')
|
12
12
|
rule 'S' => 'A'
|
13
13
|
rule 'A' => 'a A c'
|
14
14
|
rule 'A' => 'b'
|
15
15
|
end
|
16
|
-
|
17
|
-
return builder
|
18
16
|
end
|
19
17
|
end # module
|
20
18
|
# End of file
|
@@ -10,16 +10,15 @@ module GrammarAmbig01Helper
|
|
10
10
|
# Factory method. Define a grammar for a very simple language
|
11
11
|
# Grammar 3: An ambiguous arithmetic expression language
|
12
12
|
# (based on example in article on Earley's algorithm in Wikipedia)
|
13
|
-
def grammar_ambig01_builder
|
14
|
-
|
13
|
+
def grammar_ambig01_builder
|
14
|
+
Rley::Syntax::GrammarBuilder.new do
|
15
15
|
add_terminals('integer', '+', '*')
|
16
16
|
rule 'P' => 'S'
|
17
|
-
rule 'S' =>
|
18
|
-
rule 'S' =>
|
17
|
+
rule 'S' => 'S + S'
|
18
|
+
rule 'S' => 'S * S'
|
19
19
|
rule 'S' => 'L'
|
20
20
|
rule 'L' => 'integer'
|
21
21
|
end
|
22
|
-
builder
|
23
22
|
end
|
24
23
|
|
25
24
|
# Highly simplified tokenizer implementation.
|
@@ -11,17 +11,16 @@ module GrammarArrIntHelper
|
|
11
11
|
# Factory method. Creates a grammar builder for a grammar of
|
12
12
|
# array of integers.
|
13
13
|
# (based on the article about Earley's algorithm in Wikipedia)
|
14
|
-
def grammar_arr_int_builder
|
15
|
-
|
14
|
+
def grammar_arr_int_builder
|
15
|
+
Rley::Syntax::GrammarBuilder.new do
|
16
16
|
add_terminals('[', ']', ',', 'integer')
|
17
17
|
rule 'P' => 'arr'
|
18
|
-
rule 'arr' =>
|
18
|
+
rule 'arr' => '[ sequence ]'
|
19
19
|
rule 'sequence' => 'list'
|
20
20
|
rule 'sequence' => []
|
21
|
-
rule 'list' =>
|
21
|
+
rule 'list' => 'list , integer' # Left-recursive rule
|
22
22
|
rule 'list' => 'integer'
|
23
23
|
end
|
24
|
-
builder
|
25
24
|
end
|
26
25
|
|
27
26
|
# Basic tokenizer for array of integers
|
@@ -9,17 +9,16 @@ module GrammarBExprHelper
|
|
9
9
|
# Factory method. Creates a grammar builder for a basic arithmetic
|
10
10
|
# expression grammar.
|
11
11
|
# (based on the article about Earley's algorithm in Wikipedia)
|
12
|
-
def grammar_expr_builder
|
13
|
-
|
12
|
+
def grammar_expr_builder
|
13
|
+
Rley::Syntax::GrammarBuilder.new do
|
14
14
|
add_terminals('+', '*', 'integer')
|
15
15
|
rule 'P' => 'S'
|
16
|
-
rule 'S' =>
|
16
|
+
rule 'S' => 'S + M'
|
17
17
|
rule 'S' => 'M'
|
18
|
-
rule 'M' =>
|
18
|
+
rule 'M' => 'M * T'
|
19
19
|
rule 'M' => 'T'
|
20
20
|
rule 'T' => 'integer'
|
21
21
|
end
|
22
|
-
builder
|
23
22
|
end
|
24
23
|
|
25
24
|
# Basic expression tokenizer
|
@@ -11,29 +11,28 @@ module GrammarL0Helper
|
|
11
11
|
# based on Jurafky & Martin L0 language (chapter 12 of the book).
|
12
12
|
# It defines the syntax of a sentence in a language with a
|
13
13
|
# very limited syntax and lexicon in the context of airline reservation.
|
14
|
-
def grammar_l0_builder
|
15
|
-
|
14
|
+
def grammar_l0_builder
|
15
|
+
Rley::Syntax::GrammarBuilder.new do
|
16
16
|
add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
|
17
17
|
add_terminals('Determiner', 'Preposition')
|
18
|
-
rule 'S' =>
|
18
|
+
rule 'S' => 'NP VP'
|
19
19
|
rule 'NP' => 'Pronoun'
|
20
20
|
rule 'NP' => 'Proper-Noun'
|
21
|
-
rule 'NP' =>
|
22
|
-
rule 'Nominal' =>
|
21
|
+
rule 'NP' => 'Determiner Nominal'
|
22
|
+
rule 'Nominal' => 'Nominal Noun'
|
23
23
|
rule 'Nominal' => 'Noun'
|
24
24
|
rule 'VP' => 'Verb'
|
25
|
-
rule 'VP' =>
|
26
|
-
rule 'VP' =>
|
27
|
-
rule 'VP' =>
|
28
|
-
rule 'PP' =>
|
25
|
+
rule 'VP' => 'Verb NP'
|
26
|
+
rule 'VP' => 'Verb NP PP'
|
27
|
+
rule 'VP' => 'Verb PP'
|
28
|
+
rule 'PP' => 'Preposition PP'
|
29
29
|
end
|
30
|
-
builder
|
31
30
|
end
|
32
31
|
|
33
32
|
# Return the language lexicon.
|
34
33
|
# A lexicon is just a Hash with pairs of the form:
|
35
34
|
# word => terminal symbol name
|
36
|
-
def lexicon_l0
|
35
|
+
def lexicon_l0
|
37
36
|
return {
|
38
37
|
'flight' => 'Noun',
|
39
38
|
'breeze' => 'Noun',
|
@@ -12,7 +12,7 @@ class GrammarPBHelper
|
|
12
12
|
# expression based on example found in paper of
|
13
13
|
# K. Pingali and G. Bilardi:
|
14
14
|
# "A Graphical Model for Context-Free Grammar Parsing"
|
15
|
-
def grammar
|
15
|
+
def grammar
|
16
16
|
@grammar ||= begin
|
17
17
|
builder = Rley::Syntax::GrammarBuilder.new do
|
18
18
|
add_terminals('int', '+', '(', ')')
|
@@ -24,12 +24,13 @@ class GrammarPBHelper
|
|
24
24
|
builder.grammar
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
class PB_Tokenizer < BaseTokenizer
|
29
29
|
protected
|
30
30
|
|
31
|
-
|
32
|
-
|
31
|
+
# rubocop: disable Lint/DuplicateBranch
|
32
|
+
def recognize_token
|
33
|
+
if (lexeme = scanner.scan(/[()]/)) # Single characters
|
33
34
|
# Delimiters, separators => single character token
|
34
35
|
build_token(lexeme, lexeme)
|
35
36
|
elsif (lexeme = scanner.scan(/(?:\+)(?=\s|$)/)) # Isolated char
|
@@ -38,6 +39,7 @@ class GrammarPBHelper
|
|
38
39
|
build_token('int', lexeme)
|
39
40
|
end
|
40
41
|
end
|
42
|
+
# rubocop: enable Lint/DuplicateBranch
|
41
43
|
end # class
|
42
44
|
|
43
45
|
# Basic tokenizer
|
@@ -46,6 +48,5 @@ class GrammarPBHelper
|
|
46
48
|
tokenizer = PB_Tokenizer.new(aText)
|
47
49
|
tokenizer.tokens
|
48
50
|
end
|
49
|
-
|
50
51
|
end # class
|
51
52
|
# End of file
|
@@ -10,7 +10,7 @@ module GrammarSPPFHelper
|
|
10
10
|
# "SPPF-Style Parsing From Earley Recognizers" in
|
11
11
|
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
12
12
|
# contains a hidden left recursion and a cycle
|
13
|
-
def grammar_sppf_builder
|
13
|
+
def grammar_sppf_builder
|
14
14
|
builder = Rley::Syntax::GrammarBuilder.new do
|
15
15
|
add_terminals('a', 'b')
|
16
16
|
rule 'Phi' => 'S'
|
@@ -24,7 +24,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
24
24
|
|
25
25
|
context 'Initialization with argument:' do
|
26
26
|
it 'could be created with a block argument' do
|
27
|
-
expect do
|
27
|
+
expect do
|
28
28
|
GrammarBuilder.new { nil }
|
29
29
|
end.not_to raise_error
|
30
30
|
end
|
@@ -99,14 +99,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
99
99
|
expect(new_prod.lhs).to eq(subject['A'])
|
100
100
|
expect_rhs = [subject['a'], subject['A'], subject['c']]
|
101
101
|
expect(new_prod.rhs.members).to eq(expect_rhs)
|
102
|
-
|
102
|
+
|
103
103
|
# Try another syntax
|
104
104
|
subject.add_production('A' => 'a A c')
|
105
105
|
expect(subject.productions.size).to eq(3)
|
106
106
|
new_prod = subject.productions.last
|
107
107
|
expect(new_prod.lhs).to eq(subject['A'])
|
108
108
|
expect_rhs = [subject['a'], subject['A'], subject['c']]
|
109
|
-
expect(new_prod.rhs.members).to eq(expect_rhs)
|
109
|
+
expect(new_prod.rhs.members).to eq(expect_rhs)
|
110
110
|
|
111
111
|
# GrammarBuilder#rule is an alias of add_production
|
112
112
|
subject.rule('A' => ['b'])
|
@@ -162,11 +162,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
162
162
|
err = StandardError
|
163
163
|
msg = 'Useless terminal symbol(s): d.'
|
164
164
|
expect { subject.grammar }.to raise_error(err, msg)
|
165
|
-
|
165
|
+
|
166
166
|
# Add another useless terminal
|
167
167
|
subject.add_terminals('e')
|
168
168
|
msg = 'Useless terminal symbol(s): d, e.'
|
169
|
-
expect { subject.grammar }.to raise_error(err, msg)
|
169
|
+
expect { subject.grammar }.to raise_error(err, msg)
|
170
170
|
end
|
171
171
|
|
172
172
|
it 'should build a grammar with nullable nonterminals' do
|
@@ -74,7 +74,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
74
74
|
let(:nt_A) { NonTerminal.new('A') }
|
75
75
|
let(:nt_B) { NonTerminal.new('B') }
|
76
76
|
let(:nt_C) { NonTerminal.new('C') }
|
77
|
-
let(:nt_D) { NonTerminal.new('D') }
|
77
|
+
let(:nt_D) { NonTerminal.new('D') }
|
78
78
|
let(:a_) { VerbatimSymbol.new('a') }
|
79
79
|
let(:b_) { VerbatimSymbol.new('b') }
|
80
80
|
let(:c_) { VerbatimSymbol.new('c') }
|
@@ -161,7 +161,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
161
161
|
expect(subject.name2symbol['b']).to eq(b_)
|
162
162
|
expect(subject.name2symbol['c']).to eq(c_)
|
163
163
|
end
|
164
|
-
|
164
|
+
|
165
165
|
it 'should ensure that each production has a name' do
|
166
166
|
subject.rules.each do |prod|
|
167
167
|
expect(prod.name).to match(Regexp.new("#{prod.lhs.name}_\\d$"))
|
@@ -218,8 +218,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
218
218
|
partitioning = instance.non_terminals.partition(&:generative?)
|
219
219
|
expect(partitioning[0].size).to eq(2)
|
220
220
|
expect(partitioning[0]).to eq([nt_S, nt_A])
|
221
|
-
expect(partitioning[1]).to eq([nt_B, nt_C, nt_D])
|
222
|
-
|
221
|
+
expect(partitioning[1]).to eq([nt_B, nt_C, nt_D])
|
222
|
+
|
223
223
|
undefined = instance.non_terminals.select(&:undefined?)
|
224
224
|
expect(undefined).to be_empty
|
225
225
|
end
|
@@ -248,13 +248,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
248
248
|
expect(nterm).to be_nullable
|
249
249
|
end
|
250
250
|
end
|
251
|
-
|
251
|
+
|
252
252
|
it 'should mark its nullable productions' do
|
253
253
|
# Given the above productions, here are our expectations:
|
254
254
|
expectations = [true, false, false, true]
|
255
255
|
actuals = subject.rules.map(&:nullable?)
|
256
256
|
expect(actuals).to eq(expectations)
|
257
|
-
end
|
257
|
+
end
|
258
258
|
end # context
|
259
259
|
end # describe
|
260
260
|
end # module
|