rley 0.7.03 → 0.7.08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +362 -62
- data/.travis.yml +6 -7
- data/CHANGELOG.md +20 -1
- data/LICENSE.txt +1 -1
- data/README.md +6 -7
- data/Rakefile +2 -0
- data/appveyor.yml +2 -4
- data/examples/NLP/benchmark_pico_en.rb +2 -0
- data/examples/NLP/engtagger.rb +193 -188
- data/examples/NLP/nano_eng/nano_en_demo.rb +2 -0
- data/examples/NLP/nano_eng/nano_grammar.rb +7 -5
- data/examples/NLP/pico_en_demo.rb +2 -0
- data/examples/data_formats/JSON/cli_options.rb +3 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +14 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +14 -21
- data/examples/data_formats/JSON/json_demo.rb +2 -0
- data/examples/data_formats/JSON/json_grammar.rb +4 -2
- data/examples/data_formats/JSON/json_lexer.rb +10 -8
- data/examples/data_formats/JSON/json_minifier.rb +3 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +15 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +25 -37
- data/examples/general/calc_iter1/calc_demo.rb +2 -0
- data/examples/general/calc_iter1/calc_grammar.rb +4 -2
- data/examples/general/calc_iter1/calc_lexer.rb +8 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +7 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +7 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +29 -43
- data/examples/general/calc_iter2/calc_demo.rb +2 -0
- data/examples/general/calc_iter2/calc_grammar.rb +5 -3
- data/examples/general/calc_iter2/calc_lexer.rb +13 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +28 -26
- data/examples/general/left.rb +4 -2
- data/examples/general/right.rb +4 -2
- data/lib/rley.rb +2 -0
- data/lib/rley/base/base_parser.rb +2 -0
- data/lib/rley/base/dotted_item.rb +38 -41
- data/lib/rley/base/grm_items_builder.rb +2 -0
- data/lib/rley/constants.rb +5 -3
- data/lib/rley/engine.rb +22 -24
- data/lib/rley/formatter/asciitree.rb +6 -4
- data/lib/rley/formatter/base_formatter.rb +2 -0
- data/lib/rley/formatter/bracket_notation.rb +3 -8
- data/lib/rley/formatter/debug.rb +8 -6
- data/lib/rley/formatter/json.rb +4 -2
- data/lib/rley/gfg/call_edge.rb +3 -1
- data/lib/rley/gfg/edge.rb +7 -5
- data/lib/rley/gfg/end_vertex.rb +4 -6
- data/lib/rley/gfg/epsilon_edge.rb +3 -5
- data/lib/rley/gfg/grm_flow_graph.rb +31 -25
- data/lib/rley/gfg/item_vertex.rb +12 -22
- data/lib/rley/gfg/non_terminal_vertex.rb +6 -4
- data/lib/rley/gfg/return_edge.rb +2 -0
- data/lib/rley/gfg/scan_edge.rb +3 -1
- data/lib/rley/gfg/shortcut_edge.rb +4 -2
- data/lib/rley/gfg/start_vertex.rb +6 -8
- data/lib/rley/gfg/vertex.rb +47 -41
- data/lib/rley/lexical/token.rb +3 -1
- data/lib/rley/lexical/token_range.rb +8 -6
- data/lib/rley/parse_forest_visitor.rb +7 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +11 -11
- data/lib/rley/parse_rep/cst_builder.rb +7 -4
- data/lib/rley/parse_rep/parse_forest_builder.rb +36 -25
- data/lib/rley/parse_rep/parse_forest_factory.rb +5 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +18 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +15 -15
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -25
- data/lib/rley/parse_tree_visitor.rb +3 -1
- data/lib/rley/parser/error_reason.rb +9 -8
- data/lib/rley/parser/gfg_chart.rb +54 -22
- data/lib/rley/parser/gfg_earley_parser.rb +3 -1
- data/lib/rley/parser/gfg_parsing.rb +51 -31
- data/lib/rley/parser/parse_entry.rb +29 -33
- data/lib/rley/parser/parse_entry_set.rb +32 -27
- data/lib/rley/parser/parse_entry_tracker.rb +6 -4
- data/lib/rley/parser/parse_state.rb +18 -21
- data/lib/rley/parser/parse_state_tracker.rb +6 -4
- data/lib/rley/parser/parse_tracer.rb +15 -13
- data/lib/rley/parser/parse_walker_factory.rb +28 -29
- data/lib/rley/parser/state_set.rb +11 -10
- data/lib/rley/ptree/non_terminal_node.rb +10 -6
- data/lib/rley/ptree/parse_tree.rb +6 -4
- data/lib/rley/ptree/parse_tree_node.rb +7 -5
- data/lib/rley/ptree/terminal_node.rb +9 -7
- data/lib/rley/rley_error.rb +12 -10
- data/lib/rley/sppf/alternative_node.rb +8 -6
- data/lib/rley/sppf/composite_node.rb +9 -7
- data/lib/rley/sppf/epsilon_node.rb +5 -3
- data/lib/rley/sppf/leaf_node.rb +5 -3
- data/lib/rley/sppf/non_terminal_node.rb +2 -0
- data/lib/rley/sppf/parse_forest.rb +19 -17
- data/lib/rley/sppf/sppf_node.rb +9 -8
- data/lib/rley/sppf/token_node.rb +5 -3
- data/lib/rley/syntax/grammar.rb +7 -5
- data/lib/rley/syntax/grammar_builder.rb +11 -9
- data/lib/rley/syntax/grm_symbol.rb +8 -6
- data/lib/rley/syntax/literal.rb +2 -0
- data/lib/rley/syntax/non_terminal.rb +11 -15
- data/lib/rley/syntax/production.rb +13 -11
- data/lib/rley/syntax/symbol_seq.rb +10 -10
- data/lib/rley/syntax/terminal.rb +6 -5
- data/lib/rley/syntax/verbatim_symbol.rb +5 -3
- data/lib/support/base_tokenizer.rb +23 -20
- data/spec/rley/base/dotted_item_spec.rb +4 -2
- data/spec/rley/base/grm_items_builder_spec.rb +2 -0
- data/spec/rley/engine_spec.rb +47 -9
- data/spec/rley/formatter/asciitree_spec.rb +11 -9
- data/spec/rley/formatter/bracket_notation_spec.rb +16 -14
- data/spec/rley/formatter/debug_spec.rb +4 -2
- data/spec/rley/formatter/json_spec.rb +5 -3
- data/spec/rley/gfg/call_edge_spec.rb +2 -0
- data/spec/rley/gfg/edge_spec.rb +2 -0
- data/spec/rley/gfg/end_vertex_spec.rb +7 -5
- data/spec/rley/gfg/epsilon_edge_spec.rb +2 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -0
- data/spec/rley/gfg/item_vertex_spec.rb +12 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +5 -3
- data/spec/rley/gfg/return_edge_spec.rb +2 -0
- data/spec/rley/gfg/scan_edge_spec.rb +2 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +3 -1
- data/spec/rley/gfg/start_vertex_spec.rb +7 -5
- data/spec/rley/gfg/vertex_spec.rb +5 -3
- data/spec/rley/lexical/token_range_spec.rb +18 -16
- data/spec/rley/lexical/token_spec.rb +4 -2
- data/spec/rley/parse_forest_visitor_spec.rb +167 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +46 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +8 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +7 -5
- data/spec/rley/parse_rep/groucho_spec.rb +25 -25
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +28 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +8 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +4 -2
- data/spec/rley/parse_tree_visitor_spec.rb +12 -8
- data/spec/rley/parser/error_reason_spec.rb +8 -6
- data/spec/rley/parser/gfg_chart_spec.rb +17 -4
- data/spec/rley/parser/gfg_earley_parser_spec.rb +16 -11
- data/spec/rley/parser/gfg_parsing_spec.rb +41 -252
- data/spec/rley/parser/parse_entry_set_spec.rb +2 -0
- data/spec/rley/parser/parse_entry_spec.rb +21 -19
- data/spec/rley/parser/parse_state_spec.rb +7 -5
- data/spec/rley/parser/parse_tracer_spec.rb +16 -14
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -8
- data/spec/rley/parser/state_set_spec.rb +24 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +7 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +6 -4
- data/spec/rley/ptree/parse_tree_spec.rb +2 -0
- data/spec/rley/ptree/terminal_node_spec.rb +8 -6
- data/spec/rley/sppf/alternative_node_spec.rb +8 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +5 -3
- data/spec/rley/sppf/token_node_spec.rb +6 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +5 -4
- data/spec/rley/support/expectation_helper.rb +2 -0
- data/spec/rley/support/grammar_abc_helper.rb +4 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +6 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +6 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +6 -5
- data/spec/rley/support/grammar_helper.rb +2 -0
- data/spec/rley/support/grammar_l0_helper.rb +15 -16
- data/spec/rley/support/grammar_pb_helper.rb +8 -5
- data/spec/rley/support/grammar_sppf_helper.rb +3 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +7 -5
- data/spec/rley/syntax/grammar_spec.rb +8 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +3 -1
- data/spec/rley/syntax/literal_spec.rb +2 -0
- data/spec/rley/syntax/non_terminal_spec.rb +10 -8
- data/spec/rley/syntax/production_spec.rb +15 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +4 -2
- data/spec/rley/syntax/terminal_spec.rb +7 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +3 -1
- data/spec/spec_helper.rb +2 -12
- data/spec/support/base_tokenizer_spec.rb +9 -2
- metadata +21 -63
- data/.simplecov +0 -7
- data/Gemfile +0 -8
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ostruct'
|
2
4
|
require_relative '../../spec_helper'
|
3
5
|
|
@@ -31,7 +33,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
31
33
|
terminal = OpenStruct.new(name: aSymbolName)
|
32
34
|
return OpenStruct.new(lexeme: aLexeme, terminal: terminal)
|
33
35
|
end
|
34
|
-
|
36
|
+
|
35
37
|
it 'should accept the addition of subnodes' do
|
36
38
|
child1 = double('first_child')
|
37
39
|
child2 = double('second_child')
|
@@ -42,6 +44,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
42
44
|
expect(subject.subnodes).to eq([child3, child2, child1])
|
43
45
|
end
|
44
46
|
|
47
|
+
# rubocop: disable Naming/VariableNumber
|
45
48
|
it 'should provide a text representation of itself' do
|
46
49
|
# Case 1: no child
|
47
50
|
expected_text = 'VP[0, 3]'
|
@@ -59,9 +62,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
59
62
|
noun = build_token('bus', 'Noun')
|
60
63
|
child_3_1 = TerminalNode.new(noun, range(2, 3))
|
61
64
|
# We reverse the sequence of subnode addition
|
62
|
-
subject.add_subnode(child_1_2)
|
65
|
+
subject.add_subnode(child_1_2)
|
63
66
|
subject.add_subnode(child_1_1)
|
64
|
-
child_1_2.add_subnode(child_2_2)
|
67
|
+
child_1_2.add_subnode(child_2_2)
|
65
68
|
child_1_2.add_subnode(child_2_1)
|
66
69
|
child_2_2.add_subnode(child_3_1)
|
67
70
|
expected_text = <<-SNIPPET
|
@@ -74,6 +77,7 @@ VP[0, 3]
|
|
74
77
|
SNIPPET
|
75
78
|
expect(subject.to_string(0)).to eq(expected_text.chomp)
|
76
79
|
end
|
80
|
+
# rubocop: enable Naming/VariableNumber
|
77
81
|
end # context
|
78
82
|
end # describe
|
79
83
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative '../../spec_helper'
|
2
4
|
|
3
5
|
# Load the class under test
|
@@ -25,17 +27,17 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
25
27
|
expect(subject.range).to eq(sample_range)
|
26
28
|
end
|
27
29
|
end # context
|
28
|
-
|
30
|
+
|
29
31
|
context 'Initialization:' do
|
30
32
|
it 'should assign undefined range bounds' do
|
31
33
|
partial_range = { low: 0 } # High bound left undefined
|
32
34
|
instance = ParseTreeNode.new(sample_symbol, partial_range)
|
33
|
-
|
35
|
+
|
34
36
|
another = { low: 1, high: 4 } # High bound is specified
|
35
37
|
instance.range = another
|
36
38
|
expect(instance.range).to eq(low: 0, high: 4)
|
37
|
-
end
|
38
|
-
end # context
|
39
|
+
end
|
40
|
+
end # context
|
39
41
|
end # describe
|
40
42
|
end # module
|
41
43
|
end # module
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ostruct'
|
2
4
|
require_relative '../../spec_helper'
|
3
5
|
|
@@ -8,23 +10,23 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
8
10
|
module PTree # Open this namespace to avoid module qualifier prefixes
|
9
11
|
describe TerminalNode do
|
10
12
|
let(:sample_symbol) { OpenStruct.new(name: 'Noun') }
|
11
|
-
let(:sample_token) do
|
12
|
-
OpenStruct.new(lexeme: 'world', terminal: sample_symbol)
|
13
|
+
let(:sample_token) do
|
14
|
+
OpenStruct.new(lexeme: 'world', terminal: sample_symbol)
|
13
15
|
end
|
14
16
|
let(:sample_range) { double('fake-range') }
|
15
|
-
|
17
|
+
|
16
18
|
subject { TerminalNode.new(sample_token, sample_range) }
|
17
|
-
|
19
|
+
|
18
20
|
context 'Initialization:' do
|
19
21
|
it 'should be bound to a token' do
|
20
22
|
expect(subject.token).to eq(sample_token)
|
21
23
|
end
|
22
24
|
end # context
|
23
|
-
|
25
|
+
|
24
26
|
context 'Provided services:' do
|
25
27
|
it 'should provide a text representation of itself' do
|
26
28
|
expected_text = "Noun[?, ?]: 'world'"
|
27
|
-
expect(subject.to_string(0)).to eq(expected_text)
|
29
|
+
expect(subject.to_string(0)).to eq(expected_text)
|
28
30
|
end
|
29
31
|
end # context
|
30
32
|
end # describe
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ostruct'
|
2
4
|
require_relative '../../spec_helper'
|
3
5
|
|
@@ -17,7 +19,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
17
19
|
def range(low, high)
|
18
20
|
return Lexical::TokenRange.new(low: low, high: high)
|
19
21
|
end
|
20
|
-
|
22
|
+
|
21
23
|
let(:t_a) { Syntax::Terminal.new('A') }
|
22
24
|
let(:t_b) { Syntax::Terminal.new('B') }
|
23
25
|
let(:t_c) { Syntax::Terminal.new('C') }
|
@@ -27,7 +29,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
27
29
|
end
|
28
30
|
let(:sample_item) { Base::DottedItem.new(sample_prod, 3) }
|
29
31
|
let(:sample_vertex) { GFG::ItemVertex.new(sample_item) }
|
30
|
-
let(:sample_range) { range(0, 3) }
|
32
|
+
let(:sample_range) { range(0, 3) }
|
31
33
|
|
32
34
|
subject { AlternativeNode.new(sample_vertex, sample_range) }
|
33
35
|
|
@@ -35,9 +37,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
35
37
|
it 'should be created with a item vertex and a token range' do
|
36
38
|
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
37
39
|
.not_to raise_error
|
38
|
-
end
|
40
|
+
end
|
39
41
|
end
|
40
|
-
|
42
|
+
|
41
43
|
context 'Initialization:' do
|
42
44
|
it 'should know its token range' do
|
43
45
|
expect(subject.range).to eq(sample_range)
|
@@ -48,7 +50,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
48
50
|
expect(subject.subnodes).to be_empty
|
49
51
|
end
|
50
52
|
end # context
|
51
|
-
|
53
|
+
|
52
54
|
context 'Provided services:' do
|
53
55
|
it 'should accept the addition of subnodes' do
|
54
56
|
subnode1 = double('first_subnode')
|
@@ -61,7 +63,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
61
63
|
end
|
62
64
|
|
63
65
|
|
64
|
-
it 'should have a string representation' do
|
66
|
+
it 'should have a string representation' do
|
65
67
|
expect(subject.to_string(0)).to eq('Alt(sentence => A B C .)[0, 3]')
|
66
68
|
end
|
67
69
|
end # context
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ostruct'
|
2
4
|
require_relative '../../spec_helper'
|
3
5
|
|
@@ -35,12 +37,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
35
37
|
it "shouldn't have children yet" do
|
36
38
|
expect(subject.subnodes).to be_empty
|
37
39
|
end
|
38
|
-
|
40
|
+
|
39
41
|
it 'should have :and refinement' do
|
40
42
|
expect(subject.refinement).to eq(:and)
|
41
43
|
end
|
42
44
|
end # context
|
43
|
-
|
45
|
+
|
44
46
|
context 'Provided services:' do
|
45
47
|
it 'should accept the addition of subnodes' do
|
46
48
|
subnode1 = double('first_subnode')
|
@@ -55,7 +57,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
55
57
|
it 'should have a string representation' do
|
56
58
|
expect(subject.to_string(0)).to eq('VP[0, 3]')
|
57
59
|
end
|
58
|
-
|
60
|
+
|
59
61
|
it 'should return a key value of itself' do
|
60
62
|
expect(subject.key).to eq('VP[0, 3]')
|
61
63
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'ostruct'
|
2
4
|
require_relative '../../spec_helper'
|
3
5
|
|
@@ -13,8 +15,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
13
15
|
describe TokenNode do
|
14
16
|
let(:sample_symbol) { Syntax::Terminal.new('Noun') }
|
15
17
|
let(:sample_position) { Lexical::Position.new(3, 4) }
|
16
|
-
let(:sample_token) do
|
17
|
-
Lexical::Token.new('language', sample_symbol, sample_position)
|
18
|
+
let(:sample_token) do
|
19
|
+
Lexical::Token.new('language', sample_symbol, sample_position)
|
18
20
|
end
|
19
21
|
let(:sample_rank) { 3 }
|
20
22
|
|
@@ -31,13 +33,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
31
33
|
expect(subject.range.high).to eq(sample_rank + 1)
|
32
34
|
end
|
33
35
|
end # context
|
34
|
-
|
36
|
+
|
35
37
|
context 'Provided services:' do
|
36
38
|
it 'should know its string representation' do
|
37
39
|
expect(subject.to_string(0)).to eq('Noun[3, 4]')
|
38
40
|
expect(subject.inspect).to eq('Noun[3, 4]')
|
39
41
|
end
|
40
|
-
|
42
|
+
|
41
43
|
it 'should return a key value of itself' do
|
42
44
|
expect(subject.key).to eq('Noun[3, 4]')
|
43
45
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
require_relative '../../../lib/rley/lexical/token'
|
@@ -7,14 +9,13 @@ module AmbiguousGrammarHelper
|
|
7
9
|
# Factory method. Creates a grammar builder for a basic ambiguous
|
8
10
|
# expression grammar.
|
9
11
|
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
10
|
-
def grammar_builder
|
11
|
-
|
12
|
+
def grammar_builder
|
13
|
+
Rley::Syntax::GrammarBuilder.new do
|
12
14
|
add_terminals('+', 'id')
|
13
15
|
rule 'S' => 'E'
|
14
|
-
rule 'E' =>
|
16
|
+
rule 'E' => 'E + E'
|
15
17
|
rule 'E' => 'id'
|
16
18
|
end
|
17
|
-
builder
|
18
19
|
end
|
19
20
|
|
20
21
|
# Basic tokenizing method
|
@@ -1,18 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
|
4
6
|
module GrammarABCHelper
|
5
7
|
# Factory method. Creates a grammar builder for a simple grammar.
|
6
8
|
# (based on example in N. Wirth "Compiler Construction" book, p. 6)
|
7
|
-
def grammar_abc_builder
|
8
|
-
|
9
|
+
def grammar_abc_builder
|
10
|
+
Rley::Syntax::GrammarBuilder.new do
|
9
11
|
add_terminals('a', 'b', 'c')
|
10
12
|
rule 'S' => 'A'
|
11
13
|
rule 'A' => 'a A c'
|
12
14
|
rule 'A' => 'b'
|
13
15
|
end
|
14
|
-
|
15
|
-
return builder
|
16
16
|
end
|
17
17
|
end # module
|
18
18
|
# End of file
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
require_relative '../../../lib/rley/lexical/token'
|
@@ -8,16 +10,15 @@ module GrammarAmbig01Helper
|
|
8
10
|
# Factory method. Define a grammar for a very simple language
|
9
11
|
# Grammar 3: An ambiguous arithmetic expression language
|
10
12
|
# (based on example in article on Earley's algorithm in Wikipedia)
|
11
|
-
def grammar_ambig01_builder
|
12
|
-
|
13
|
+
def grammar_ambig01_builder
|
14
|
+
Rley::Syntax::GrammarBuilder.new do
|
13
15
|
add_terminals('integer', '+', '*')
|
14
16
|
rule 'P' => 'S'
|
15
|
-
rule 'S' =>
|
16
|
-
rule 'S' =>
|
17
|
+
rule 'S' => 'S + S'
|
18
|
+
rule 'S' => 'S * S'
|
17
19
|
rule 'S' => 'L'
|
18
20
|
rule 'L' => 'integer'
|
19
21
|
end
|
20
|
-
builder
|
21
22
|
end
|
22
23
|
|
23
24
|
# Highly simplified tokenizer implementation.
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'strscan'
|
2
4
|
|
3
5
|
# Load the builder class
|
@@ -9,17 +11,16 @@ module GrammarArrIntHelper
|
|
9
11
|
# Factory method. Creates a grammar builder for a grammar of
|
10
12
|
# array of integers.
|
11
13
|
# (based on the article about Earley's algorithm in Wikipedia)
|
12
|
-
def grammar_arr_int_builder
|
13
|
-
|
14
|
+
def grammar_arr_int_builder
|
15
|
+
Rley::Syntax::GrammarBuilder.new do
|
14
16
|
add_terminals('[', ']', ',', 'integer')
|
15
17
|
rule 'P' => 'arr'
|
16
|
-
rule 'arr' =>
|
18
|
+
rule 'arr' => '[ sequence ]'
|
17
19
|
rule 'sequence' => 'list'
|
18
20
|
rule 'sequence' => []
|
19
|
-
rule 'list' =>
|
21
|
+
rule 'list' => 'list , integer' # Left-recursive rule
|
20
22
|
rule 'list' => 'integer'
|
21
23
|
end
|
22
|
-
builder
|
23
24
|
end
|
24
25
|
|
25
26
|
# Basic tokenizer for array of integers
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
require_relative '../../../lib/rley/lexical/token'
|
@@ -7,17 +9,16 @@ module GrammarBExprHelper
|
|
7
9
|
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
10
|
# expression grammar.
|
9
11
|
# (based on the article about Earley's algorithm in Wikipedia)
|
10
|
-
def grammar_expr_builder
|
11
|
-
|
12
|
+
def grammar_expr_builder
|
13
|
+
Rley::Syntax::GrammarBuilder.new do
|
12
14
|
add_terminals('+', '*', 'integer')
|
13
15
|
rule 'P' => 'S'
|
14
|
-
rule 'S' =>
|
16
|
+
rule 'S' => 'S + M'
|
15
17
|
rule 'S' => 'M'
|
16
|
-
rule 'M' =>
|
18
|
+
rule 'M' => 'M * T'
|
17
19
|
rule 'M' => 'T'
|
18
20
|
rule 'T' => 'integer'
|
19
21
|
end
|
20
|
-
builder
|
21
22
|
end
|
22
23
|
|
23
24
|
# Basic expression tokenizer
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
require_relative '../../../lib/rley/lexical/token'
|
@@ -9,29 +11,28 @@ module GrammarL0Helper
|
|
9
11
|
# based on Jurafky & Martin L0 language (chapter 12 of the book).
|
10
12
|
# It defines the syntax of a sentence in a language with a
|
11
13
|
# very limited syntax and lexicon in the context of airline reservation.
|
12
|
-
def grammar_l0_builder
|
13
|
-
|
14
|
+
def grammar_l0_builder
|
15
|
+
Rley::Syntax::GrammarBuilder.new do
|
14
16
|
add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
|
15
17
|
add_terminals('Determiner', 'Preposition')
|
16
|
-
rule 'S' =>
|
18
|
+
rule 'S' => 'NP VP'
|
17
19
|
rule 'NP' => 'Pronoun'
|
18
20
|
rule 'NP' => 'Proper-Noun'
|
19
|
-
rule 'NP' =>
|
20
|
-
rule 'Nominal' =>
|
21
|
+
rule 'NP' => 'Determiner Nominal'
|
22
|
+
rule 'Nominal' => 'Nominal Noun'
|
21
23
|
rule 'Nominal' => 'Noun'
|
22
24
|
rule 'VP' => 'Verb'
|
23
|
-
rule 'VP' =>
|
24
|
-
rule 'VP' =>
|
25
|
-
rule 'VP' =>
|
26
|
-
rule 'PP' =>
|
25
|
+
rule 'VP' => 'Verb NP'
|
26
|
+
rule 'VP' => 'Verb NP PP'
|
27
|
+
rule 'VP' => 'Verb PP'
|
28
|
+
rule 'PP' => 'Preposition PP'
|
27
29
|
end
|
28
|
-
builder
|
29
30
|
end
|
30
31
|
|
31
32
|
# Return the language lexicon.
|
32
33
|
# A lexicon is just a Hash with pairs of the form:
|
33
34
|
# word => terminal symbol name
|
34
|
-
def lexicon_l0
|
35
|
+
def lexicon_l0
|
35
36
|
return {
|
36
37
|
'flight' => 'Noun',
|
37
38
|
'breeze' => 'Noun',
|
@@ -76,13 +77,11 @@ module GrammarL0Helper
|
|
76
77
|
word = scanner.scan(/\S+/)
|
77
78
|
break unless word
|
78
79
|
|
79
|
-
|
80
|
-
if
|
81
|
-
raise StandardError, "Word '#{word}' not found in lexicon"
|
82
|
-
end
|
80
|
+
term = lexicon_l0[word]
|
81
|
+
raise StandardError, "Word '#{word}' not found in lexicon" if term.nil?
|
83
82
|
|
84
83
|
pos = Rley::Lexical::Position.new(1, curr_pos + 1)
|
85
|
-
tokens << Rley::Lexical::Token.new(word,
|
84
|
+
tokens << Rley::Lexical::Token.new(word, term, pos)
|
86
85
|
end
|
87
86
|
|
88
87
|
return tokens
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Load the builder class
|
2
4
|
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
5
|
require_relative '../../../lib/support/base_tokenizer'
|
@@ -10,7 +12,7 @@ class GrammarPBHelper
|
|
10
12
|
# expression based on example found in paper of
|
11
13
|
# K. Pingali and G. Bilardi:
|
12
14
|
# "A Graphical Model for Context-Free Grammar Parsing"
|
13
|
-
def grammar
|
15
|
+
def grammar
|
14
16
|
@grammar ||= begin
|
15
17
|
builder = Rley::Syntax::GrammarBuilder.new do
|
16
18
|
add_terminals('int', '+', '(', ')')
|
@@ -22,12 +24,13 @@ class GrammarPBHelper
|
|
22
24
|
builder.grammar
|
23
25
|
end
|
24
26
|
end
|
25
|
-
|
27
|
+
|
26
28
|
class PB_Tokenizer < BaseTokenizer
|
27
29
|
protected
|
28
30
|
|
29
|
-
|
30
|
-
|
31
|
+
# rubocop: disable Lint/DuplicateBranch
|
32
|
+
def recognize_token
|
33
|
+
if (lexeme = scanner.scan(/[()]/)) # Single characters
|
31
34
|
# Delimiters, separators => single character token
|
32
35
|
build_token(lexeme, lexeme)
|
33
36
|
elsif (lexeme = scanner.scan(/(?:\+)(?=\s|$)/)) # Isolated char
|
@@ -36,6 +39,7 @@ class GrammarPBHelper
|
|
36
39
|
build_token('int', lexeme)
|
37
40
|
end
|
38
41
|
end
|
42
|
+
# rubocop: enable Lint/DuplicateBranch
|
39
43
|
end # class
|
40
44
|
|
41
45
|
# Basic tokenizer
|
@@ -44,6 +48,5 @@ class GrammarPBHelper
|
|
44
48
|
tokenizer = PB_Tokenizer.new(aText)
|
45
49
|
tokenizer.tokens
|
46
50
|
end
|
47
|
-
|
48
51
|
end # class
|
49
52
|
# End of file
|