rley 0.3.04 → 0.3.05
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +7 -3
- data/CHANGELOG.md +3 -0
- data/Rakefile +30 -30
- data/examples/parsers/parsing_L0.rb +1 -1
- data/examples/parsers/parsing_L1.rb +1 -1
- data/examples/parsers/parsing_abc.rb +1 -1
- data/examples/parsers/parsing_ambig.rb +1 -1
- data/examples/parsers/parsing_another.rb +1 -1
- data/examples/parsers/parsing_b_expr.rb +1 -1
- data/examples/parsers/parsing_err_expr.rb +1 -1
- data/examples/parsers/parsing_groucho.rb +1 -1
- data/examples/parsers/parsing_right_recursive.rb +1 -1
- data/examples/parsers/parsing_tricky.rb +1 -1
- data/lib/rley/constants.rb +2 -2
- data/lib/rley/formatter/base_formatter.rb +0 -2
- data/lib/rley/formatter/debug.rb +0 -2
- data/lib/rley/formatter/json.rb +1 -3
- data/lib/rley/gfg/call_edge.rb +31 -30
- data/lib/rley/gfg/edge.rb +22 -23
- data/lib/rley/gfg/end_vertex.rb +22 -24
- data/lib/rley/gfg/epsilon_edge.rb +20 -21
- data/lib/rley/gfg/grm_flow_graph.rb +39 -39
- data/lib/rley/gfg/item_vertex.rb +16 -17
- data/lib/rley/gfg/non_terminal_vertex.rb +3 -4
- data/lib/rley/gfg/return_edge.rb +32 -31
- data/lib/rley/gfg/scan_edge.rb +25 -26
- data/lib/rley/gfg/shortcut_edge.rb +25 -26
- data/lib/rley/gfg/start_vertex.rb +0 -2
- data/lib/rley/gfg/vertex.rb +8 -8
- data/lib/rley/parse_forest_visitor.rb +113 -115
- data/lib/rley/parse_tree_visitor.rb +0 -2
- data/lib/rley/parser/base_parser.rb +27 -27
- data/lib/rley/parser/chart.rb +14 -14
- data/lib/rley/parser/dotted_item.rb +33 -33
- data/lib/rley/parser/earley_parser.rb +6 -6
- data/lib/rley/parser/gfg_chart.rb +8 -15
- data/lib/rley/parser/gfg_earley_parser.rb +15 -13
- data/lib/rley/parser/gfg_parsing.rb +26 -22
- data/lib/rley/parser/grm_items_builder.rb +3 -2
- data/lib/rley/parser/parse_entry.rb +3 -9
- data/lib/rley/parser/parse_entry_set.rb +14 -19
- data/lib/rley/parser/parse_entry_tracker.rb +56 -56
- data/lib/rley/parser/parse_forest_builder.rb +215 -214
- data/lib/rley/parser/parse_forest_factory.rb +57 -56
- data/lib/rley/parser/parse_state.rb +8 -11
- data/lib/rley/parser/parse_state_tracker.rb +56 -56
- data/lib/rley/parser/parse_tracer.rb +3 -3
- data/lib/rley/parser/parse_tree_builder.rb +10 -10
- data/lib/rley/parser/parse_walker_factory.rb +30 -33
- data/lib/rley/parser/parsing.rb +8 -8
- data/lib/rley/parser/state_set.rb +23 -26
- data/lib/rley/ptree/non_terminal_node.rb +1 -1
- data/lib/rley/ptree/token_range.rb +2 -2
- data/lib/rley/sppf/alternative_node.rb +32 -34
- data/lib/rley/sppf/composite_node.rb +27 -27
- data/lib/rley/sppf/epsilon_node.rb +26 -27
- data/lib/rley/sppf/leaf_node.rb +11 -12
- data/lib/rley/sppf/non_terminal_node.rb +37 -38
- data/lib/rley/sppf/sppf_node.rb +1 -1
- data/lib/rley/sppf/token_node.rb +29 -29
- data/lib/rley/syntax/grammar.rb +1 -3
- data/lib/rley/syntax/grammar_builder.rb +8 -8
- data/lib/rley/syntax/non_terminal.rb +2 -4
- data/lib/rley/syntax/production.rb +3 -3
- data/lib/rley/syntax/symbol_seq.rb +1 -1
- data/spec/rley/gfg/call_edge_spec.rb +50 -51
- data/spec/rley/gfg/edge_spec.rb +33 -33
- data/spec/rley/gfg/end_vertex_spec.rb +26 -27
- data/spec/rley/gfg/epsilon_edge_spec.rb +25 -25
- data/spec/rley/gfg/grm_flow_graph_spec.rb +1 -1
- data/spec/rley/gfg/item_vertex_spec.rb +3 -4
- data/spec/rley/gfg/return_edge_spec.rb +51 -51
- data/spec/rley/gfg/scan_edge_spec.rb +32 -30
- data/spec/rley/gfg/shortcut_edge_spec.rb +1 -1
- data/spec/rley/gfg/vertex_spec.rb +3 -3
- data/spec/rley/parse_forest_visitor_spec.rb +239 -238
- data/spec/rley/parser/dotted_item_spec.rb +1 -1
- data/spec/rley/parser/earley_parser_spec.rb +16 -16
- data/spec/rley/parser/gfg_earley_parser_spec.rb +30 -31
- data/spec/rley/parser/gfg_parsing_spec.rb +11 -10
- data/spec/rley/parser/grm_items_builder_spec.rb +2 -2
- data/spec/rley/parser/parse_entry_set_spec.rb +4 -4
- data/spec/rley/parser/parse_entry_spec.rb +0 -2
- data/spec/rley/parser/parse_forest_builder_spec.rb +82 -57
- data/spec/rley/parser/parse_forest_factory_spec.rb +84 -82
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -9
- data/spec/rley/parser/parsing_spec.rb +0 -1
- data/spec/rley/sppf/alternative_node_spec.rb +2 -2
- data/spec/rley/sppf/non_terminal_node_spec.rb +0 -1
- data/spec/rley/support/ambiguous_grammar_helper.rb +1 -1
- data/spec/rley/support/expectation_helper.rb +37 -36
- data/spec/rley/support/grammar_abc_helper.rb +17 -17
- data/spec/rley/support/grammar_b_expr_helper.rb +40 -39
- data/spec/rley/support/grammar_helper.rb +2 -1
- data/spec/rley/support/{grammar_L0_helper.rb → grammar_l0_helper.rb} +82 -81
- data/spec/rley/support/grammar_sppf_helper.rb +24 -25
- data/spec/rley/syntax/grammar_spec.rb +1 -1
- metadata +2 -2
@@ -1,82 +1,84 @@
|
|
1
|
-
require_relative '../../spec_helper'
|
2
|
-
|
3
|
-
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
4
|
-
|
5
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
6
|
-
require_relative '../support/grammar_helper'
|
7
|
-
require_relative '../support/expectation_helper'
|
8
|
-
|
9
|
-
# Load the class under test
|
10
|
-
require_relative '../../../lib/rley/parser/parse_forest_factory'
|
11
|
-
|
12
|
-
module Rley # Open this namespace to avoid module qualifier prefixes
|
13
|
-
module Parser
|
14
|
-
describe ParseForestFactory do
|
15
|
-
include GrammarHelper # Mix-in with token factory method
|
16
|
-
include ExpectationHelper # Mix-in with expectation on parse entry sets
|
17
|
-
|
18
|
-
let(:sample_grammar) do
|
19
|
-
# Grammar based on paper from Elisabeth Scott
|
20
|
-
# "SPPF=Style Parsing From Earley Recognizers" in
|
21
|
-
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
22
|
-
# contains a hidden left recursion and a cycle
|
23
|
-
builder = Syntax::GrammarBuilder.new
|
24
|
-
builder.add_terminals('a', 'b')
|
25
|
-
builder.add_production('Phi' =>
|
26
|
-
builder.add_production('S' => %w
|
27
|
-
builder.add_production('S' => %w
|
28
|
-
builder.add_production('A' => 'a')
|
29
|
-
builder.add_production('A' => %w
|
30
|
-
builder.add_production('B' => [])
|
31
|
-
builder.add_production('T' => %w(
|
32
|
-
builder.grammar
|
33
|
-
end
|
34
|
-
|
35
|
-
let(:sample_tokens) do
|
36
|
-
build_token_sequence(%w(a b b b), sample_grammar)
|
37
|
-
end
|
38
|
-
|
39
|
-
let(:sample_result) do
|
40
|
-
parser = Parser::GFGEarleyParser.new(sample_grammar)
|
41
|
-
parser.parse(sample_tokens)
|
42
|
-
end
|
43
|
-
|
44
|
-
|
45
|
-
subject do
|
46
|
-
ParseForestFactory.new(sample_result)
|
47
|
-
end
|
48
|
-
|
49
|
-
# Emit a text representation of the current path.
|
50
|
-
def path_to_s()
|
51
|
-
text_parts = subject.curr_path.map
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
=
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
=
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
end #
|
82
|
-
#
|
1
|
+
require_relative '../../spec_helper'
|
2
|
+
|
3
|
+
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
4
|
+
|
5
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
6
|
+
require_relative '../support/grammar_helper'
|
7
|
+
require_relative '../support/expectation_helper'
|
8
|
+
|
9
|
+
# Load the class under test
|
10
|
+
require_relative '../../../lib/rley/parser/parse_forest_factory'
|
11
|
+
|
12
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
13
|
+
module Parser
|
14
|
+
describe ParseForestFactory do
|
15
|
+
include GrammarHelper # Mix-in with token factory method
|
16
|
+
include ExpectationHelper # Mix-in with expectation on parse entry sets
|
17
|
+
|
18
|
+
let(:sample_grammar) do
|
19
|
+
# Grammar based on paper from Elisabeth Scott
|
20
|
+
# "SPPF=Style Parsing From Earley Recognizers" in
|
21
|
+
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
22
|
+
# contains a hidden left recursion and a cycle
|
23
|
+
builder = Syntax::GrammarBuilder.new
|
24
|
+
builder.add_terminals('a', 'b')
|
25
|
+
builder.add_production('Phi' => 'S')
|
26
|
+
builder.add_production('S' => %w(A T))
|
27
|
+
builder.add_production('S' => %w(a T))
|
28
|
+
builder.add_production('A' => 'a')
|
29
|
+
builder.add_production('A' => %w(B A))
|
30
|
+
builder.add_production('B' => [])
|
31
|
+
builder.add_production('T' => %w(b b b))
|
32
|
+
builder.grammar
|
33
|
+
end
|
34
|
+
|
35
|
+
let(:sample_tokens) do
|
36
|
+
build_token_sequence(%w(a b b b), sample_grammar)
|
37
|
+
end
|
38
|
+
|
39
|
+
let(:sample_result) do
|
40
|
+
parser = Parser::GFGEarleyParser.new(sample_grammar)
|
41
|
+
parser.parse(sample_tokens)
|
42
|
+
end
|
43
|
+
|
44
|
+
|
45
|
+
subject do
|
46
|
+
ParseForestFactory.new(sample_result)
|
47
|
+
end
|
48
|
+
|
49
|
+
# Emit a text representation of the current path.
|
50
|
+
def path_to_s()
|
51
|
+
text_parts = subject.curr_path.map do |path_element|
|
52
|
+
path_element.to_string(0)
|
53
|
+
end
|
54
|
+
return text_parts.join('/')
|
55
|
+
end
|
56
|
+
|
57
|
+
|
58
|
+
context 'Initialization:' do
|
59
|
+
it 'should be created with a GFGParsing' do
|
60
|
+
expect { ParseForestFactory.new(sample_result) }.not_to raise_error
|
61
|
+
end
|
62
|
+
|
63
|
+
it 'should know the parse result' do
|
64
|
+
expect(subject.parsing).to eq(sample_result)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
context 'Parse forest construction' do
|
69
|
+
it 'should build a parse forest' do
|
70
|
+
forest = subject.build_parse_forest
|
71
|
+
expect(forest).to be_kind_of(SPPF::ParseForest)
|
72
|
+
=begin
|
73
|
+
require 'yaml'
|
74
|
+
require_relative '../../../exp/lab/forest_representation'
|
75
|
+
File.open("forest.yml", "w") { |f| YAML.dump(forest, f) }
|
76
|
+
pen = ForestRepresentation.new
|
77
|
+
pen.generate_graph(forest, File.open("forest.dot", "w"))
|
78
|
+
=end
|
79
|
+
end
|
80
|
+
end # context
|
81
|
+
end # describe
|
82
|
+
end # module
|
83
|
+
end # module
|
84
|
+
# End of file
|
@@ -45,13 +45,13 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
45
45
|
# contains a hidden left recursion and a cycle
|
46
46
|
builder = Syntax::GrammarBuilder.new
|
47
47
|
builder.add_terminals('a', 'b')
|
48
|
-
builder.add_production('Phi' =>
|
49
|
-
builder.add_production('S' => %w
|
50
|
-
builder.add_production('S' => %w
|
48
|
+
builder.add_production('Phi' => 'S')
|
49
|
+
builder.add_production('S' => %w(A T))
|
50
|
+
builder.add_production('S' => %w(a T))
|
51
51
|
builder.add_production('A' => 'a')
|
52
|
-
builder.add_production('A' => %w
|
52
|
+
builder.add_production('A' => %w(B A))
|
53
53
|
builder.add_production('B' => [])
|
54
|
-
builder.add_production('T' => %w(
|
54
|
+
builder.add_production('T' => %w(b b b))
|
55
55
|
builder.grammar
|
56
56
|
end
|
57
57
|
|
@@ -77,7 +77,9 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
77
77
|
|
78
78
|
context 'Parse graph traversal:' do
|
79
79
|
it 'should create an Enumerator as a walker' do
|
80
|
-
|
80
|
+
entry = accept_entry
|
81
|
+
index = accept_index
|
82
|
+
expect(subject.build_walker(entry, index)).to be_kind_of(Enumerator)
|
81
83
|
end
|
82
84
|
|
83
85
|
it 'should return the accepting parse entry in the first place' do
|
@@ -247,11 +249,10 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
247
249
|
walker = subject.build_walker(accept_entry, accept_index)
|
248
250
|
37.times { walker.next }
|
249
251
|
|
250
|
-
expect{ walker.next }.to raise_error(StopIteration)
|
252
|
+
expect { walker.next }.to raise_error(StopIteration)
|
251
253
|
end
|
252
|
-
|
253
254
|
end # context
|
254
255
|
end # describe
|
255
256
|
end # module
|
256
257
|
end # module
|
257
|
-
# End of file
|
258
|
+
# End of file
|
@@ -33,7 +33,8 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
33
33
|
|
34
34
|
context 'Construction:' do
|
35
35
|
it 'should be created with a item vertex and a token range' do
|
36
|
-
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
36
|
+
expect { AlternativeNode.new(sample_vertex, sample_range) }
|
37
|
+
.not_to raise_error
|
37
38
|
end
|
38
39
|
end
|
39
40
|
|
@@ -64,7 +65,6 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
64
65
|
expect(subject.to_string(0)).to eq('Alt(sentence => A B C .)[0, 3]')
|
65
66
|
end
|
66
67
|
end # context
|
67
|
-
|
68
68
|
end # describe
|
69
69
|
end # module
|
70
70
|
end # module
|
@@ -1,36 +1,37 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/parser/token'
|
3
|
-
|
4
|
-
|
5
|
-
# Mixin module implementing expectation helper methods.
|
6
|
-
module ExpectationHelper
|
7
|
-
# Helper method. Compare the data from all the parse entries
|
8
|
-
# of a given ParseEntrySet with an array of expectation strings.
|
9
|
-
def compare_entry_texts(anEntrySet, expectations)
|
10
|
-
(0...expectations.size).each do |i|
|
11
|
-
expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
# Helper method. Compare the antecedents from all the parse entries
|
16
|
-
# of a given ParseEntrySet at given position with a Hash of the form:
|
17
|
-
# consequent label => [ antecedent label(s) ]
|
18
|
-
def check_antecedence(aParsing, aPosition, expectations)
|
19
|
-
entry_set = aParsing.chart[aPosition]
|
20
|
-
|
21
|
-
expectations.each do |consequent_label, antecedent_labels|
|
22
|
-
consequent = entry_set.entries.find do |entry|
|
23
|
-
entry.to_s == consequent_label
|
24
|
-
end
|
25
|
-
actual_antecedents = aParsing.antecedence.fetch(consequent)
|
26
|
-
expect(actual_antecedents.map(&:to_s)).to eq(antecedent_labels)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
|
31
|
-
def expected_terminals(anEntrySet, termNames)
|
32
|
-
terminals = anEntrySet.expected_terminals
|
33
|
-
actual_names = terminals.map(&:name)
|
34
|
-
expect(actual_names.sort).to eq(termNames.sort)
|
35
|
-
end
|
36
|
-
end # module
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/parser/token'
|
3
|
+
|
4
|
+
|
5
|
+
# Mixin module implementing expectation helper methods.
|
6
|
+
module ExpectationHelper
|
7
|
+
# Helper method. Compare the data from all the parse entries
|
8
|
+
# of a given ParseEntrySet with an array of expectation strings.
|
9
|
+
def compare_entry_texts(anEntrySet, expectations)
|
10
|
+
(0...expectations.size).each do |i|
|
11
|
+
expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
# Helper method. Compare the antecedents from all the parse entries
|
16
|
+
# of a given ParseEntrySet at given position with a Hash of the form:
|
17
|
+
# consequent label => [ antecedent label(s) ]
|
18
|
+
def check_antecedence(aParsing, aPosition, expectations)
|
19
|
+
entry_set = aParsing.chart[aPosition]
|
20
|
+
|
21
|
+
expectations.each do |consequent_label, antecedent_labels|
|
22
|
+
consequent = entry_set.entries.find do |entry|
|
23
|
+
entry.to_s == consequent_label
|
24
|
+
end
|
25
|
+
actual_antecedents = aParsing.antecedence.fetch(consequent)
|
26
|
+
expect(actual_antecedents.map(&:to_s)).to eq(antecedent_labels)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
def expected_terminals(anEntrySet, termNames)
|
32
|
+
terminals = anEntrySet.expected_terminals
|
33
|
+
actual_names = terminals.map(&:name)
|
34
|
+
expect(actual_names.sort).to eq(termNames.sort)
|
35
|
+
end
|
36
|
+
end # module
|
37
|
+
# End of file
|
@@ -1,17 +1,17 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
#
|
7
|
-
|
8
|
-
|
9
|
-
builder
|
10
|
-
builder.
|
11
|
-
builder.add_production('
|
12
|
-
builder.add_production('A' =>
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
+
|
4
|
+
module GrammarABCHelper
|
5
|
+
# Factory method. Creates a grammar builder for a simple grammar.
|
6
|
+
# (based on example in N. Wirth "Compiler Construction" book, p. 6)
|
7
|
+
def grammar_abc_builder()
|
8
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
9
|
+
builder.add_terminals('a', 'b', 'c')
|
10
|
+
builder.add_production('S' => 'A')
|
11
|
+
builder.add_production('A' => %w(a A c))
|
12
|
+
builder.add_production('A' => 'b')
|
13
|
+
|
14
|
+
return builder
|
15
|
+
end
|
16
|
+
end # module
|
17
|
+
# End of file
|
@@ -1,39 +1,40 @@
|
|
1
|
-
# Load the builder class
|
2
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
-
require_relative '../../../lib/rley/parser/token'
|
4
|
-
|
5
|
-
|
6
|
-
module GrammarBExprHelper
|
7
|
-
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
|
-
# expression grammar.
|
9
|
-
# (based on the article about Earley's algorithm in Wikipedia)
|
10
|
-
def grammar_expr_builder()
|
11
|
-
builder = Rley::Syntax::GrammarBuilder.new
|
12
|
-
builder.add_terminals('+', '*', 'integer')
|
13
|
-
builder.add_production('P' => 'S')
|
14
|
-
builder.add_production('S' => %w(S + M))
|
15
|
-
builder.add_production('S' => 'M')
|
16
|
-
builder.add_production('M' => %w(M * T))
|
17
|
-
builder.add_production('M' => 'T')
|
18
|
-
builder.add_production('T' => 'integer')
|
19
|
-
builder
|
20
|
-
end
|
21
|
-
|
22
|
-
# Basic expression tokenizer
|
23
|
-
def expr_tokenizer(aText, aGrammar)
|
24
|
-
tokens = aText.scan(/\S+/).map do |lexeme|
|
25
|
-
case lexeme
|
26
|
-
when '+', '*'
|
27
|
-
terminal = aGrammar.name2symbol[lexeme]
|
28
|
-
when /^[-+]?\d+$/
|
29
|
-
terminal = aGrammar.name2symbol['integer']
|
30
|
-
else
|
31
|
-
msg = "Unknown input text '#{lexeme}'"
|
32
|
-
|
33
|
-
end
|
34
|
-
Rley::Parser::Token.new(lexeme, terminal)
|
35
|
-
end
|
36
|
-
|
37
|
-
return tokens
|
38
|
-
end
|
39
|
-
end # module
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
+
require_relative '../../../lib/rley/parser/token'
|
4
|
+
|
5
|
+
|
6
|
+
module GrammarBExprHelper
|
7
|
+
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
|
+
# expression grammar.
|
9
|
+
# (based on the article about Earley's algorithm in Wikipedia)
|
10
|
+
def grammar_expr_builder()
|
11
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
12
|
+
builder.add_terminals('+', '*', 'integer')
|
13
|
+
builder.add_production('P' => 'S')
|
14
|
+
builder.add_production('S' => %w(S + M))
|
15
|
+
builder.add_production('S' => 'M')
|
16
|
+
builder.add_production('M' => %w(M * T))
|
17
|
+
builder.add_production('M' => 'T')
|
18
|
+
builder.add_production('T' => 'integer')
|
19
|
+
builder
|
20
|
+
end
|
21
|
+
|
22
|
+
# Basic expression tokenizer
|
23
|
+
def expr_tokenizer(aText, aGrammar)
|
24
|
+
tokens = aText.scan(/\S+/).map do |lexeme|
|
25
|
+
case lexeme
|
26
|
+
when '+', '*'
|
27
|
+
terminal = aGrammar.name2symbol[lexeme]
|
28
|
+
when /^[-+]?\d+$/
|
29
|
+
terminal = aGrammar.name2symbol['integer']
|
30
|
+
else
|
31
|
+
msg = "Unknown input text '#{lexeme}'"
|
32
|
+
raise StandardError, msg
|
33
|
+
end
|
34
|
+
Rley::Parser::Token.new(lexeme, terminal)
|
35
|
+
end
|
36
|
+
|
37
|
+
return tokens
|
38
|
+
end
|
39
|
+
end # module
|
40
|
+
# End of file
|