rley 0.3.04 → 0.3.05
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -3
- data/CHANGELOG.md +3 -0
- data/Rakefile +30 -30
- data/examples/parsers/parsing_L0.rb +1 -1
- data/examples/parsers/parsing_L1.rb +1 -1
- data/examples/parsers/parsing_abc.rb +1 -1
- data/examples/parsers/parsing_ambig.rb +1 -1
- data/examples/parsers/parsing_another.rb +1 -1
- data/examples/parsers/parsing_b_expr.rb +1 -1
- data/examples/parsers/parsing_err_expr.rb +1 -1
- data/examples/parsers/parsing_groucho.rb +1 -1
- data/examples/parsers/parsing_right_recursive.rb +1 -1
- data/examples/parsers/parsing_tricky.rb +1 -1
- data/lib/rley/constants.rb +2 -2
- data/lib/rley/formatter/base_formatter.rb +0 -2
- data/lib/rley/formatter/debug.rb +0 -2
- data/lib/rley/formatter/json.rb +1 -3
- data/lib/rley/gfg/call_edge.rb +31 -30
- data/lib/rley/gfg/edge.rb +22 -23
- data/lib/rley/gfg/end_vertex.rb +22 -24
- data/lib/rley/gfg/epsilon_edge.rb +20 -21
- data/lib/rley/gfg/grm_flow_graph.rb +39 -39
- data/lib/rley/gfg/item_vertex.rb +16 -17
- data/lib/rley/gfg/non_terminal_vertex.rb +3 -4
- data/lib/rley/gfg/return_edge.rb +32 -31
- data/lib/rley/gfg/scan_edge.rb +25 -26
- data/lib/rley/gfg/shortcut_edge.rb +25 -26
- data/lib/rley/gfg/start_vertex.rb +0 -2
- data/lib/rley/gfg/vertex.rb +8 -8
- data/lib/rley/parse_forest_visitor.rb +113 -115
- data/lib/rley/parse_tree_visitor.rb +0 -2
- data/lib/rley/parser/base_parser.rb +27 -27
- data/lib/rley/parser/chart.rb +14 -14
- data/lib/rley/parser/dotted_item.rb +33 -33
- data/lib/rley/parser/earley_parser.rb +6 -6
- data/lib/rley/parser/gfg_chart.rb +8 -15
- data/lib/rley/parser/gfg_earley_parser.rb +15 -13
- data/lib/rley/parser/gfg_parsing.rb +26 -22
- data/lib/rley/parser/grm_items_builder.rb +3 -2
- data/lib/rley/parser/parse_entry.rb +3 -9
- data/lib/rley/parser/parse_entry_set.rb +14 -19
- data/lib/rley/parser/parse_entry_tracker.rb +56 -56
- data/lib/rley/parser/parse_forest_builder.rb +215 -214
- data/lib/rley/parser/parse_forest_factory.rb +57 -56
- data/lib/rley/parser/parse_state.rb +8 -11
- data/lib/rley/parser/parse_state_tracker.rb +56 -56
- data/lib/rley/parser/parse_tracer.rb +3 -3
- data/lib/rley/parser/parse_tree_builder.rb +10 -10
- data/lib/rley/parser/parse_walker_factory.rb +30 -33
- data/lib/rley/parser/parsing.rb +8 -8
- data/lib/rley/parser/state_set.rb +23 -26
- data/lib/rley/ptree/non_terminal_node.rb +1 -1
- data/lib/rley/ptree/token_range.rb +2 -2
- data/lib/rley/sppf/alternative_node.rb +32 -34
- data/lib/rley/sppf/composite_node.rb +27 -27
- data/lib/rley/sppf/epsilon_node.rb +26 -27
- data/lib/rley/sppf/leaf_node.rb +11 -12
- data/lib/rley/sppf/non_terminal_node.rb +37 -38
- data/lib/rley/sppf/sppf_node.rb +1 -1
- data/lib/rley/sppf/token_node.rb +29 -29
- data/lib/rley/syntax/grammar.rb +1 -3
- data/lib/rley/syntax/grammar_builder.rb +8 -8
- data/lib/rley/syntax/non_terminal.rb +2 -4
- data/lib/rley/syntax/production.rb +3 -3
- data/lib/rley/syntax/symbol_seq.rb +1 -1
- data/spec/rley/gfg/call_edge_spec.rb +50 -51
- data/spec/rley/gfg/edge_spec.rb +33 -33
- data/spec/rley/gfg/end_vertex_spec.rb +26 -27
- data/spec/rley/gfg/epsilon_edge_spec.rb +25 -25
- data/spec/rley/gfg/grm_flow_graph_spec.rb +1 -1
- data/spec/rley/gfg/item_vertex_spec.rb +3 -4
- data/spec/rley/gfg/return_edge_spec.rb +51 -51
- data/spec/rley/gfg/scan_edge_spec.rb +32 -30
- data/spec/rley/gfg/shortcut_edge_spec.rb +1 -1
- data/spec/rley/gfg/vertex_spec.rb +3 -3
- data/spec/rley/parse_forest_visitor_spec.rb +239 -238
- data/spec/rley/parser/dotted_item_spec.rb +1 -1
- data/spec/rley/parser/earley_parser_spec.rb +16 -16
- data/spec/rley/parser/gfg_earley_parser_spec.rb +30 -31
- data/spec/rley/parser/gfg_parsing_spec.rb +11 -10
- data/spec/rley/parser/grm_items_builder_spec.rb +2 -2
- data/spec/rley/parser/parse_entry_set_spec.rb +4 -4
- data/spec/rley/parser/parse_entry_spec.rb +0 -2
- data/spec/rley/parser/parse_forest_builder_spec.rb +82 -57
- data/spec/rley/parser/parse_forest_factory_spec.rb +84 -82
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -9
- data/spec/rley/parser/parsing_spec.rb +0 -1
- data/spec/rley/sppf/alternative_node_spec.rb +2 -2
- data/spec/rley/sppf/non_terminal_node_spec.rb +0 -1
- data/spec/rley/support/ambiguous_grammar_helper.rb +1 -1
- data/spec/rley/support/expectation_helper.rb +37 -36
- data/spec/rley/support/grammar_abc_helper.rb +17 -17
- data/spec/rley/support/grammar_b_expr_helper.rb +40 -39
- data/spec/rley/support/grammar_helper.rb +2 -1
- data/spec/rley/support/{grammar_L0_helper.rb → grammar_l0_helper.rb} +82 -81
- data/spec/rley/support/grammar_sppf_helper.rb +24 -25
- data/spec/rley/syntax/grammar_spec.rb +1 -1
- metadata +2 -2
@@ -1,82 +1,84 @@
|
|
1
|
-
require_relative '../../spec_helper'
|
2
|
-
|
3
|
-
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
4
|
-
|
5
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
6
|
-
require_relative '../support/grammar_helper'
|
7
|
-
require_relative '../support/expectation_helper'
|
8
|
-
|
9
|
-
# Load the class under test
|
10
|
-
require_relative '../../../lib/rley/parser/parse_forest_factory'
|
11
|
-
|
12
|
-
module Rley # Open this namespace to avoid module qualifier prefixes
|
13
|
-
module Parser
|
14
|
-
describe ParseForestFactory do
|
15
|
-
include GrammarHelper # Mix-in with token factory method
|
16
|
-
include ExpectationHelper # Mix-in with expectation on parse entry sets
|
17
|
-
|
18
|
-
let(:sample_grammar) do
|
19
|
-
# Grammar based on paper from Elisabeth Scott
|
20
|
-
# "SPPF=Style Parsing From Earley Recognizers" in
|
21
|
-
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
22
|
-
# contains a hidden left recursion and a cycle
|
23
|
-
builder = Syntax::GrammarBuilder.new
|
24
|
-
builder.add_terminals('a', 'b')
|
25
|
-
builder.add_production('Phi' =>
|
26
|
-
builder.add_production('S' => %w
|
27
|
-
builder.add_production('S' => %w
|
28
|
-
builder.add_production('A' => 'a')
|
29
|
-
builder.add_production('A' => %w
|
30
|
-
builder.add_production('B' => [])
|
31
|
-
builder.add_production('T' => %w(
|
32
|
-
builder.grammar
|
33
|
-
end
|
34
|
-
|
35
|
-
let(:sample_tokens) do
|
36
|
-
build_token_sequence(%w(a b b b), sample_grammar)
|
37
|
-
end
|
38
|
-
|
39
|
-
let(:sample_result) do
|
40
|
-
parser = Parser::GFGEarleyParser.new(sample_grammar)
|
41
|
-
parser.parse(sample_tokens)
|
42
|
-
end
|
43
|
-
|
44
|
-
|
45
|
-
subject do
|
46
|
-
ParseForestFactory.new(sample_result)
|
47
|
-
end
|
48
|
-
|
49
|
-
# Emit a text representation of the current path.
|
50
|
-
def path_to_s()
|
51
|
-
text_parts = subject.curr_path.map
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
=
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
=
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
end #
|
82
|
-
#
|
1
|
+
require_relative '../../spec_helper'
|
2
|
+
|
3
|
+
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
4
|
+
|
5
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
6
|
+
require_relative '../support/grammar_helper'
|
7
|
+
require_relative '../support/expectation_helper'
|
8
|
+
|
9
|
+
# Load the class under test
|
10
|
+
require_relative '../../../lib/rley/parser/parse_forest_factory'
|
11
|
+
|
12
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
13
|
+
module Parser
|
14
|
+
describe ParseForestFactory do
|
15
|
+
include GrammarHelper # Mix-in with token factory method
|
16
|
+
include ExpectationHelper # Mix-in with expectation on parse entry sets
|
17
|
+
|
18
|
+
let(:sample_grammar) do
|
19
|
+
# Grammar based on paper from Elisabeth Scott
|
20
|
+
# "SPPF=Style Parsing From Earley Recognizers" in
|
21
|
+
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
22
|
+
# contains a hidden left recursion and a cycle
|
23
|
+
builder = Syntax::GrammarBuilder.new
|
24
|
+
builder.add_terminals('a', 'b')
|
25
|
+
builder.add_production('Phi' => 'S')
|
26
|
+
builder.add_production('S' => %w(A T))
|
27
|
+
builder.add_production('S' => %w(a T))
|
28
|
+
builder.add_production('A' => 'a')
|
29
|
+
builder.add_production('A' => %w(B A))
|
30
|
+
builder.add_production('B' => [])
|
31
|
+
builder.add_production('T' => %w(b b b))
|
32
|
+
builder.grammar
|
33
|
+
end
|
34
|
+
|
35
|
+
let(:sample_tokens) do
|
36
|
+
build_token_sequence(%w(a b b b), sample_grammar)
|
37
|
+
end
|
38
|
+
|
39
|
+
let(:sample_result) do
|
40
|
+
parser = Parser::GFGEarleyParser.new(sample_grammar)
|
41
|
+
parser.parse(sample_tokens)
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
subject do
|
46
|
+
ParseForestFactory.new(sample_result)
|
47
|
+
end
|
48
|
+
|
49
|
+
# Emit a text representation of the current path.
|
50
|
+
def path_to_s()
|
51
|
+
text_parts = subject.curr_path.map do |path_element|
|
52
|
+
path_element.to_string(0)
|
53
|
+
end
|
54
|
+
return text_parts.join('/')
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
context 'Initialization:' do
|
59
|
+
it 'should be created with a GFGParsing' do
|
60
|
+
expect { ParseForestFactory.new(sample_result) }.not_to raise_error
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'should know the parse result' do
|
64
|
+
expect(subject.parsing).to eq(sample_result)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'Parse forest construction' do
|
69
|
+
it 'should build a parse forest' do
|
70
|
+
forest = subject.build_parse_forest
|
71
|
+
expect(forest).to be_kind_of(SPPF::ParseForest)
|
72
|
+
=begin
|
73
|
+
require 'yaml'
|
74
|
+
require_relative '../../../exp/lab/forest_representation'
|
75
|
+
File.open("forest.yml", "w") { |f| YAML.dump(forest, f) }
|
76
|
+
pen = ForestRepresentation.new
|
77
|
+
pen.generate_graph(forest, File.open("forest.dot", "w"))
|
78
|
+
=end
|
79
|
+
end
|
80
|
+
end # context
|
81
|
+
end # describe
|
82
|
+
end # module
|
83
|
+
end # module
|
84
|
+
# End of file
|
@@ -45,13 +45,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
45
45
|
# contains a hidden left recursion and a cycle
|
46
46
|
builder = Syntax::GrammarBuilder.new
|
47
47
|
builder.add_terminals('a', 'b')
|
48
|
-
builder.add_production('Phi' =>
|
49
|
-
builder.add_production('S' => %w
|
50
|
-
builder.add_production('S' => %w
|
48
|
+
builder.add_production('Phi' => 'S')
|
49
|
+
builder.add_production('S' => %w(A T))
|
50
|
+
builder.add_production('S' => %w(a T))
|
51
51
|
builder.add_production('A' => 'a')
|
52
|
-
builder.add_production('A' => %w
|
52
|
+
builder.add_production('A' => %w(B A))
|
53
53
|
builder.add_production('B' => [])
|
54
|
-
builder.add_production('T' => %w(
|
54
|
+
builder.add_production('T' => %w(b b b))
|
55
55
|
builder.grammar
|
56
56
|
end
|
57
57
|
|
@@ -77,7 +77,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
77
77
|
|
78
78
|
context 'Parse graph traversal:' do
|
79
79
|
it 'should create an Enumerator as a walker' do
|
80
|
-
|
80
|
+
entry = accept_entry
|
81
|
+
index = accept_index
|
82
|
+
expect(subject.build_walker(entry, index)).to be_kind_of(Enumerator)
|
81
83
|
end
|
82
84
|
|
83
85
|
it 'should return the accepting parse entry in the first place' do
|
@@ -247,11 +249,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
247
249
|
walker = subject.build_walker(accept_entry, accept_index)
|
248
250
|
37.times { walker.next }
|
249
251
|
|
250
|
-
expect{ walker.next }.to raise_error(StopIteration)
|
252
|
+
expect { walker.next }.to raise_error(StopIteration)
|
251
253
|
end
|
252
|
-
|
253
254
|
end # context
|
254
255
|
end # describe
|
255
256
|
end # module
|
256
257
|
end # module
|
257
|
-
# End of file
|
258
|
+
# End of file
|
@@ -33,7 +33,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
33
33
|
|
34
34
|
context 'Construction:' do
|
35
35
|
it 'should be created with a item vertex and a token range' do
|
36
|
-
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
36
|
+
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
37
|
+
.not_to raise_error
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
@@ -64,7 +65,6 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
64
65
|
expect(subject.to_string(0)).to eq('Alt(sentence => A B C .)[0, 3]')
|
65
66
|
end
|
66
67
|
end # context
|
67
|
-
|
68
68
|
end # describe
|
69
69
|
end # module
|
70
70
|
end # module
|
@@ -1,36 +1,37 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/parser/token'
|
3
|
-
|
4
|
-
|
5
|
-
# Mixin module implementing expectation helper methods.
|
6
|
-
module ExpectationHelper
|
7
|
-
# Helper method. Compare the data from all the parse entries
|
8
|
-
# of a given ParseEntrySet with an array of expectation strings.
|
9
|
-
def compare_entry_texts(anEntrySet, expectations)
|
10
|
-
(0...expectations.size).each do |i|
|
11
|
-
expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
# Helper method. Compare the antecedents from all the parse entries
|
16
|
-
# of a given ParseEntrySet at given position with a Hash of the form:
|
17
|
-
# consequent label => [ antecedent label(s) ]
|
18
|
-
def check_antecedence(aParsing, aPosition, expectations)
|
19
|
-
entry_set = aParsing.chart[aPosition]
|
20
|
-
|
21
|
-
expectations.each do |consequent_label, antecedent_labels|
|
22
|
-
consequent = entry_set.entries.find do |entry|
|
23
|
-
entry.to_s == consequent_label
|
24
|
-
end
|
25
|
-
actual_antecedents = aParsing.antecedence.fetch(consequent)
|
26
|
-
expect(actual_antecedents.map(&:to_s)).to eq(antecedent_labels)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
|
31
|
-
def expected_terminals(anEntrySet, termNames)
|
32
|
-
terminals = anEntrySet.expected_terminals
|
33
|
-
actual_names = terminals.map(&:name)
|
34
|
-
expect(actual_names.sort).to eq(termNames.sort)
|
35
|
-
end
|
36
|
-
end # module
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/parser/token'
|
3
|
+
|
4
|
+
|
5
|
+
# Mixin module implementing expectation helper methods.
|
6
|
+
module ExpectationHelper
|
7
|
+
# Helper method. Compare the data from all the parse entries
|
8
|
+
# of a given ParseEntrySet with an array of expectation strings.
|
9
|
+
def compare_entry_texts(anEntrySet, expectations)
|
10
|
+
(0...expectations.size).each do |i|
|
11
|
+
expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# Helper method. Compare the antecedents from all the parse entries
|
16
|
+
# of a given ParseEntrySet at given position with a Hash of the form:
|
17
|
+
# consequent label => [ antecedent label(s) ]
|
18
|
+
def check_antecedence(aParsing, aPosition, expectations)
|
19
|
+
entry_set = aParsing.chart[aPosition]
|
20
|
+
|
21
|
+
expectations.each do |consequent_label, antecedent_labels|
|
22
|
+
consequent = entry_set.entries.find do |entry|
|
23
|
+
entry.to_s == consequent_label
|
24
|
+
end
|
25
|
+
actual_antecedents = aParsing.antecedence.fetch(consequent)
|
26
|
+
expect(actual_antecedents.map(&:to_s)).to eq(antecedent_labels)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def expected_terminals(anEntrySet, termNames)
|
32
|
+
terminals = anEntrySet.expected_terminals
|
33
|
+
actual_names = terminals.map(&:name)
|
34
|
+
expect(actual_names.sort).to eq(termNames.sort)
|
35
|
+
end
|
36
|
+
end # module
|
37
|
+
# End of file
|
@@ -1,17 +1,17 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
#
|
7
|
-
|
8
|
-
|
9
|
-
builder
|
10
|
-
builder.
|
11
|
-
builder.add_production('
|
12
|
-
builder.add_production('A' =>
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
+
|
4
|
+
module GrammarABCHelper
|
5
|
+
# Factory method. Creates a grammar builder for a simple grammar.
|
6
|
+
# (based on example in N. Wirth "Compiler Construction" book, p. 6)
|
7
|
+
def grammar_abc_builder()
|
8
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
9
|
+
builder.add_terminals('a', 'b', 'c')
|
10
|
+
builder.add_production('S' => 'A')
|
11
|
+
builder.add_production('A' => %w(a A c))
|
12
|
+
builder.add_production('A' => 'b')
|
13
|
+
|
14
|
+
return builder
|
15
|
+
end
|
16
|
+
end # module
|
17
|
+
# End of file
|
@@ -1,39 +1,40 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
-
require_relative '../../../lib/rley/parser/token'
|
4
|
-
|
5
|
-
|
6
|
-
module GrammarBExprHelper
|
7
|
-
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
|
-
# expression grammar.
|
9
|
-
# (based on the article about Earley's algorithm in Wikipedia)
|
10
|
-
def grammar_expr_builder()
|
11
|
-
builder = Rley::Syntax::GrammarBuilder.new
|
12
|
-
builder.add_terminals('+', '*', 'integer')
|
13
|
-
builder.add_production('P' => 'S')
|
14
|
-
builder.add_production('S' => %w(S + M))
|
15
|
-
builder.add_production('S' => 'M')
|
16
|
-
builder.add_production('M' => %w(M * T))
|
17
|
-
builder.add_production('M' => 'T')
|
18
|
-
builder.add_production('T' => 'integer')
|
19
|
-
builder
|
20
|
-
end
|
21
|
-
|
22
|
-
# Basic expression tokenizer
|
23
|
-
def expr_tokenizer(aText, aGrammar)
|
24
|
-
tokens = aText.scan(/\S+/).map do |lexeme|
|
25
|
-
case lexeme
|
26
|
-
when '+', '*'
|
27
|
-
terminal = aGrammar.name2symbol[lexeme]
|
28
|
-
when /^[-+]?\d+$/
|
29
|
-
terminal = aGrammar.name2symbol['integer']
|
30
|
-
else
|
31
|
-
msg = "Unknown input text '#{lexeme}'"
|
32
|
-
|
33
|
-
end
|
34
|
-
Rley::Parser::Token.new(lexeme, terminal)
|
35
|
-
end
|
36
|
-
|
37
|
-
return tokens
|
38
|
-
end
|
39
|
-
end # module
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
+
require_relative '../../../lib/rley/parser/token'
|
4
|
+
|
5
|
+
|
6
|
+
module GrammarBExprHelper
|
7
|
+
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
|
+
# expression grammar.
|
9
|
+
# (based on the article about Earley's algorithm in Wikipedia)
|
10
|
+
def grammar_expr_builder()
|
11
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
12
|
+
builder.add_terminals('+', '*', 'integer')
|
13
|
+
builder.add_production('P' => 'S')
|
14
|
+
builder.add_production('S' => %w(S + M))
|
15
|
+
builder.add_production('S' => 'M')
|
16
|
+
builder.add_production('M' => %w(M * T))
|
17
|
+
builder.add_production('M' => 'T')
|
18
|
+
builder.add_production('T' => 'integer')
|
19
|
+
builder
|
20
|
+
end
|
21
|
+
|
22
|
+
# Basic expression tokenizer
|
23
|
+
def expr_tokenizer(aText, aGrammar)
|
24
|
+
tokens = aText.scan(/\S+/).map do |lexeme|
|
25
|
+
case lexeme
|
26
|
+
when '+', '*'
|
27
|
+
terminal = aGrammar.name2symbol[lexeme]
|
28
|
+
when /^[-+]?\d+$/
|
29
|
+
terminal = aGrammar.name2symbol['integer']
|
30
|
+
else
|
31
|
+
msg = "Unknown input text '#{lexeme}'"
|
32
|
+
raise StandardError, msg
|
33
|
+
end
|
34
|
+
Rley::Parser::Token.new(lexeme, terminal)
|
35
|
+
end
|
36
|
+
|
37
|
+
return tokens
|
38
|
+
end
|
39
|
+
end # module
|
40
|
+
# End of file
|