rley 0.2.15 → 0.3.00
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/gfg/call_edge.rb +30 -0
- data/lib/rley/gfg/edge.rb +4 -0
- data/lib/rley/gfg/end_vertex.rb +1 -1
- data/lib/rley/gfg/epsilon_edge.rb +0 -4
- data/lib/rley/gfg/grm_flow_graph.rb +32 -7
- data/lib/rley/gfg/item_vertex.rb +71 -25
- data/lib/rley/gfg/non_terminal_vertex.rb +10 -1
- data/lib/rley/gfg/return_edge.rb +31 -0
- data/lib/rley/gfg/scan_edge.rb +2 -1
- data/lib/rley/gfg/shortcut_edge.rb +26 -0
- data/lib/rley/gfg/start_vertex.rb +2 -2
- data/lib/rley/gfg/vertex.rb +27 -1
- data/lib/rley/parse_forest_visitor.rb +115 -0
- data/lib/rley/parser/base_parser.rb +27 -0
- data/lib/rley/parser/dotted_item.rb +11 -0
- data/lib/rley/parser/earley_parser.rb +3 -15
- data/lib/rley/parser/gfg_chart.rb +106 -0
- data/lib/rley/parser/gfg_earley_parser.rb +139 -0
- data/lib/rley/parser/gfg_parsing.rb +384 -0
- data/lib/rley/parser/parse_entry.rb +148 -0
- data/lib/rley/parser/parse_entry_set.rb +104 -0
- data/lib/rley/parser/parse_entry_tracker.rb +56 -0
- data/lib/rley/parser/parse_forest_builder.rb +229 -0
- data/lib/rley/parser/parse_forest_factory.rb +54 -0
- data/lib/rley/parser/parse_walker_factory.rb +237 -0
- data/lib/rley/ptree/token_range.rb +14 -1
- data/lib/rley/sppf/alternative_node.rb +34 -0
- data/lib/rley/sppf/composite_node.rb +27 -0
- data/lib/rley/sppf/epsilon_node.rb +27 -0
- data/lib/rley/sppf/leaf_node.rb +12 -0
- data/lib/rley/sppf/non_terminal_node.rb +38 -0
- data/lib/rley/sppf/parse_forest.rb +48 -0
- data/lib/rley/sppf/sppf_node.rb +24 -0
- data/lib/rley/sppf/token_node.rb +29 -0
- data/lib/rley/syntax/grammar_builder.rb +16 -12
- data/lib/rley/syntax/grm_symbol.rb +6 -0
- data/lib/rley/syntax/terminal.rb +5 -0
- data/spec/rley/gfg/call_edge_spec.rb +51 -0
- data/spec/rley/gfg/end_vertex_spec.rb +1 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +24 -2
- data/spec/rley/gfg/item_vertex_spec.rb +75 -6
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +14 -0
- data/spec/rley/gfg/return_edge_spec.rb +51 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +43 -0
- data/spec/rley/gfg/vertex_spec.rb +52 -37
- data/spec/rley/parse_forest_visitor_spec.rb +238 -0
- data/spec/rley/parser/dotted_item_spec.rb +29 -8
- data/spec/rley/parser/gfg_chart_spec.rb +138 -0
- data/spec/rley/parser/gfg_earley_parser_spec.rb +918 -0
- data/spec/rley/parser/gfg_parsing_spec.rb +565 -0
- data/spec/rley/parser/parse_entry_set_spec.rb +179 -0
- data/spec/rley/parser/parse_entry_spec.rb +208 -0
- data/spec/rley/parser/parse_forest_builder_spec.rb +382 -0
- data/spec/rley/parser/parse_forest_factory_spec.rb +81 -0
- data/spec/rley/parser/parse_walker_factory_spec.rb +235 -0
- data/spec/rley/parser/state_set_spec.rb +4 -0
- data/spec/rley/sppf/alternative_node_spec.rb +72 -0
- data/spec/rley/sppf/antecedence_graph.rb +87 -0
- data/spec/rley/sppf/forest_representation.rb +136 -0
- data/spec/rley/sppf/gfg_representation.rb +111 -0
- data/spec/rley/sppf/non_terminal_node_spec.rb +64 -0
- data/spec/rley/support/ambiguous_grammar_helper.rb +36 -36
- data/spec/rley/support/expectation_helper.rb +36 -0
- data/spec/rley/support/grammar_helper.rb +28 -0
- data/spec/rley/support/grammar_sppf_helper.rb +25 -0
- data/spec/rley/syntax/grammar_builder_spec.rb +5 -0
- data/spec/rley/syntax/non_terminal_spec.rb +4 -0
- data/spec/rley/syntax/terminal_spec.rb +4 -0
- metadata +58 -2
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
require 'set'
|
|
2
|
+
|
|
3
|
+
# Mix-in module that generates a Graphviz's DOT file
|
|
4
|
+
# that represents a parse forest.
|
|
5
|
+
class ForestRepresentation
|
|
6
|
+
|
|
7
|
+
def generate_graph(aPForest, aFile)
|
|
8
|
+
heading = build_heading()
|
|
9
|
+
aFile.puts(heading)
|
|
10
|
+
|
|
11
|
+
fill_graph(aPForest, aFile)
|
|
12
|
+
|
|
13
|
+
trailing = build_trailing()
|
|
14
|
+
aFile.puts(trailing)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def build_heading()
|
|
20
|
+
text = <<-END_STRING
|
|
21
|
+
digraph gfg {
|
|
22
|
+
size="7,9.5";
|
|
23
|
+
page="8.5,11";
|
|
24
|
+
ratio = fill;
|
|
25
|
+
|
|
26
|
+
END_STRING
|
|
27
|
+
|
|
28
|
+
return text
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def build_trailing()
|
|
32
|
+
return '}'
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def fill_graph(aPForest, aFile)
|
|
36
|
+
visitees = Set.new
|
|
37
|
+
visit_node(aPForest.root, aFile, visitees)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def visit_node(aNode, aFile, visitees)
|
|
41
|
+
return if visitees.include?(aNode)
|
|
42
|
+
visitees << aNode
|
|
43
|
+
aFile.puts %Q( node_#{aNode.object_id}[shape=box, fontsize=18.0, label="#{aNode.to_string(0)}"];)
|
|
44
|
+
|
|
45
|
+
if aNode.kind_of?(Rley::SPPF::CompositeNode)
|
|
46
|
+
aNode.subnodes.each do |snode|
|
|
47
|
+
# puts snode.to_string(0)
|
|
48
|
+
next unless snode
|
|
49
|
+
visit_node(snode, aFile, visitees)
|
|
50
|
+
aFile.puts %Q( node_#{aNode.object_id} -> node_#{snode.object_id};)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
=begin
|
|
56
|
+
def fill_graph(aGFGraph, aFile)
|
|
57
|
+
all_vertices = aGFGraph.vertices.dup
|
|
58
|
+
(itemized, endpoints) = all_vertices.partition do |vertex|
|
|
59
|
+
vertex.is_a?(Rley::GFG::ItemVertex)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Group start/end nodes by non-terminal symbol
|
|
63
|
+
group_endings = endpoints.group_by { |endpoint| endpoint.non_terminal }
|
|
64
|
+
|
|
65
|
+
# Group item vertices by lhs non-terminal symbol
|
|
66
|
+
group_items = itemized.group_by { |vertex| vertex.lhs }
|
|
67
|
+
|
|
68
|
+
aFile.puts ''
|
|
69
|
+
group_endings.each_pair do |nonterm, nodes|
|
|
70
|
+
text = <<-END_STRING
|
|
71
|
+
subgraph cluster_#{nonterm} {
|
|
72
|
+
color = transparent;
|
|
73
|
+
END_STRING
|
|
74
|
+
aFile.puts text
|
|
75
|
+
aFile.puts ' // Define the start and end nodes'
|
|
76
|
+
nodes.each do |vertex|
|
|
77
|
+
# Emit the start/end nodes
|
|
78
|
+
aFile.puts %Q( node_#{vertex.object_id}[shape=box, fontsize=18.0, label="#{vertex.label}"];)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# Create sub-clusters by production
|
|
82
|
+
subnodes = group_items[nonterm]
|
|
83
|
+
subclusters = subnodes.group_by { |vertex| vertex.dotted_item.production }
|
|
84
|
+
subclusters.each_pair do |prod, vertices|
|
|
85
|
+
aFile.puts ''
|
|
86
|
+
aFile.puts cluster_heading(prod)
|
|
87
|
+
vertices.each do |vertex|
|
|
88
|
+
aFile.puts %Q( node_#{vertex.object_id}[label="#{vertex.label}"];)
|
|
89
|
+
end
|
|
90
|
+
aFile.puts cluster_trailing(prod)
|
|
91
|
+
end
|
|
92
|
+
aFile.puts ' }'
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
aFile.puts ''
|
|
96
|
+
aFile.puts ' // Draw the edges'
|
|
97
|
+
aGFGraph.vertices.each do |from_vertex|
|
|
98
|
+
from_vertex.edges.each do |anEdge|
|
|
99
|
+
if from_vertex.is_a?(Rley::GFG::EndVertex)
|
|
100
|
+
to_dotted_item = anEdge.successor.dotted_item
|
|
101
|
+
label = "RET_#{to_dotted_item.production.object_id}_#{to_dotted_item.prev_position}"
|
|
102
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=red, style=dashed, arrowhead=onormal,label=#{label}];"
|
|
103
|
+
else
|
|
104
|
+
if anEdge.is_a?(Rley::GFG::ScanEdge)
|
|
105
|
+
aFile.puts %Q( node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[fontsize=18.0, label="#{anEdge.terminal}"];)
|
|
106
|
+
else
|
|
107
|
+
if anEdge.successor.is_a?(Rley::GFG::StartVertex)
|
|
108
|
+
from_dotted_item = from_vertex.dotted_item
|
|
109
|
+
label = "CALL_#{from_dotted_item.production.object_id}_#{from_dotted_item.position}"
|
|
110
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=green, label=#{label}];"
|
|
111
|
+
else
|
|
112
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id};"
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def cluster_heading(anObject)
|
|
122
|
+
text = <<-END_STRING
|
|
123
|
+
subgraph cluster_#{anObject.object_id} {
|
|
124
|
+
style = rounded;
|
|
125
|
+
color = blue;
|
|
126
|
+
END_STRING
|
|
127
|
+
|
|
128
|
+
return text
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def cluster_trailing(anObject)
|
|
132
|
+
return ' }'
|
|
133
|
+
end
|
|
134
|
+
=end
|
|
135
|
+
|
|
136
|
+
end # class
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Mix-in module that generates a Graphviz's DOT file
|
|
2
|
+
# that represents the precedence graph of parse entries.
|
|
3
|
+
class GFGRepresentation
|
|
4
|
+
|
|
5
|
+
def generate_graph(aParsing, aFile)
|
|
6
|
+
heading = build_heading()
|
|
7
|
+
aFile.puts(heading)
|
|
8
|
+
|
|
9
|
+
fill_graph(aParsing.gf_graph, aFile)
|
|
10
|
+
|
|
11
|
+
trailing = build_trailing()
|
|
12
|
+
aFile.puts(trailing)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def build_heading()
|
|
18
|
+
text = <<-END_STRING
|
|
19
|
+
digraph gfg {
|
|
20
|
+
size="7,9.5";
|
|
21
|
+
page="8.5,11";
|
|
22
|
+
ratio = fill;
|
|
23
|
+
END_STRING
|
|
24
|
+
|
|
25
|
+
return text
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def build_trailing()
|
|
29
|
+
return '}'
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def fill_graph(aGFGraph, aFile)
|
|
33
|
+
all_vertices = aGFGraph.vertices.dup
|
|
34
|
+
(itemized, endpoints) = all_vertices.partition do |vertex|
|
|
35
|
+
vertex.is_a?(Rley::GFG::ItemVertex)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Group start/end nodes by non-terminal symbol
|
|
39
|
+
group_endings = endpoints.group_by { |endpoint| endpoint.non_terminal }
|
|
40
|
+
|
|
41
|
+
# Group item vertices by lhs non-terminal symbol
|
|
42
|
+
group_items = itemized.group_by { |vertex| vertex.lhs }
|
|
43
|
+
|
|
44
|
+
aFile.puts ''
|
|
45
|
+
group_endings.each_pair do |nonterm, nodes|
|
|
46
|
+
text = <<-END_STRING
|
|
47
|
+
subgraph cluster_#{nonterm} {
|
|
48
|
+
color = transparent;
|
|
49
|
+
END_STRING
|
|
50
|
+
aFile.puts text
|
|
51
|
+
aFile.puts ' // Define the start and end nodes'
|
|
52
|
+
nodes.each do |vertex|
|
|
53
|
+
# Emit the start/end nodes
|
|
54
|
+
aFile.puts %Q( node_#{vertex.object_id}[shape=box, fontsize=18.0, label="#{vertex.label}"];)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Create sub-clusters by production
|
|
58
|
+
subnodes = group_items[nonterm]
|
|
59
|
+
subclusters = subnodes.group_by { |vertex| vertex.dotted_item.production }
|
|
60
|
+
subclusters.each_pair do |prod, vertices|
|
|
61
|
+
aFile.puts ''
|
|
62
|
+
aFile.puts cluster_heading(prod)
|
|
63
|
+
vertices.each do |vertex|
|
|
64
|
+
aFile.puts %Q( node_#{vertex.object_id}[label="#{vertex.label}"];)
|
|
65
|
+
end
|
|
66
|
+
aFile.puts cluster_trailing(prod)
|
|
67
|
+
end
|
|
68
|
+
aFile.puts ' }'
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
aFile.puts ''
|
|
72
|
+
aFile.puts ' // Draw the edges'
|
|
73
|
+
aGFGraph.vertices.each do |from_vertex|
|
|
74
|
+
from_vertex.edges.each do |anEdge|
|
|
75
|
+
if from_vertex.is_a?(Rley::GFG::EndVertex)
|
|
76
|
+
to_dotted_item = anEdge.successor.dotted_item
|
|
77
|
+
label = "RET_#{to_dotted_item.production.object_id}_#{to_dotted_item.prev_position}"
|
|
78
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=red, style=dashed, arrowhead=onormal,label=#{label}];"
|
|
79
|
+
else
|
|
80
|
+
if anEdge.is_a?(Rley::GFG::ScanEdge)
|
|
81
|
+
aFile.puts %Q( node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[fontsize=18.0, label="#{anEdge.terminal}"];)
|
|
82
|
+
else
|
|
83
|
+
if anEdge.successor.is_a?(Rley::GFG::StartVertex)
|
|
84
|
+
from_dotted_item = from_vertex.dotted_item
|
|
85
|
+
label = "CALL_#{from_dotted_item.production.object_id}_#{from_dotted_item.position}"
|
|
86
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id}[color=green, label=#{label}];"
|
|
87
|
+
else
|
|
88
|
+
aFile.puts " node_#{from_vertex.object_id}->node_#{anEdge.successor.object_id};"
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def cluster_heading(anObject)
|
|
98
|
+
text = <<-END_STRING
|
|
99
|
+
subgraph cluster_#{anObject.object_id} {
|
|
100
|
+
style = rounded;
|
|
101
|
+
color = blue;
|
|
102
|
+
END_STRING
|
|
103
|
+
|
|
104
|
+
return text
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def cluster_trailing(anObject)
|
|
108
|
+
return ' }'
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
end # module
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
require 'ostruct'
|
|
2
|
+
require_relative '../../spec_helper'
|
|
3
|
+
|
|
4
|
+
require_relative '../../../lib/rley/syntax/non_terminal'
|
|
5
|
+
require_relative '../../../lib/rley/ptree/token_range'
|
|
6
|
+
|
|
7
|
+
# Load the class under test
|
|
8
|
+
require_relative '../../../lib/rley/sppf/non_terminal_node'
|
|
9
|
+
|
|
10
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
|
11
|
+
module SPPF # Open this namespace to avoid module qualifier prefixes
|
|
12
|
+
describe NonTerminalNode do
|
|
13
|
+
# Factory method. Generate a range from its boundary values.
|
|
14
|
+
def range(low, high)
|
|
15
|
+
return PTree::TokenRange.new(low: low, high: high)
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
let(:sample_symbol) do
|
|
19
|
+
Syntax::NonTerminal.new('VP')
|
|
20
|
+
end
|
|
21
|
+
let(:sample_range) { range(0, 3) }
|
|
22
|
+
|
|
23
|
+
subject { NonTerminalNode.new(sample_symbol, sample_range) }
|
|
24
|
+
|
|
25
|
+
context 'Initialization:' do
|
|
26
|
+
it 'should know its non-terminal symbol' do
|
|
27
|
+
expect(subject.symbol).to eq(sample_symbol)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
it 'should know its token range' do
|
|
31
|
+
expect(subject.range).to eq(sample_range)
|
|
32
|
+
expect(subject.origin).to eq(sample_range.low)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it "shouldn't have children yet" do
|
|
36
|
+
expect(subject.subnodes).to be_empty
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
it 'should have :and refinement' do
|
|
40
|
+
expect(subject.refinement).to eq(:and)
|
|
41
|
+
end
|
|
42
|
+
end # context
|
|
43
|
+
|
|
44
|
+
context 'Provided services:' do
|
|
45
|
+
it 'should accept the addition of subnodes' do
|
|
46
|
+
subnode1 = double('first_subnode')
|
|
47
|
+
subnode2 = double('second_subnode')
|
|
48
|
+
subnode3 = double('third_subnode')
|
|
49
|
+
expect { subject.add_subnode(subnode1) }.not_to raise_error
|
|
50
|
+
subject.add_subnode(subnode2)
|
|
51
|
+
subject.add_subnode(subnode3)
|
|
52
|
+
expect(subject.subnodes).to eq([subnode3, subnode2, subnode1])
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
it 'should have a string representation' do
|
|
56
|
+
expect(subject.to_string(0)).to eq('VP[0, 3]')
|
|
57
|
+
end
|
|
58
|
+
end # context
|
|
59
|
+
|
|
60
|
+
end # describe
|
|
61
|
+
end # module
|
|
62
|
+
end # module
|
|
63
|
+
|
|
64
|
+
# End of file
|
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
# Load the builder class
|
|
2
|
-
require_relative '../../../lib/rley/syntax/grammar_builder'
|
|
3
|
-
require_relative '../../../lib/rley/parser/token'
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
module AmbiguousGrammarHelper
|
|
7
|
-
# Factory method. Creates a grammar builder for a basic ambiguous
|
|
8
|
-
# expression grammar.
|
|
9
|
-
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
|
10
|
-
def grammar_builder()
|
|
11
|
-
builder = Rley::Syntax::GrammarBuilder.new
|
|
12
|
-
builder.add_terminals('+', 'id')
|
|
13
|
-
builder.add_production('S' => 'E')
|
|
14
|
-
builder.add_production('E' => %w(E + E))
|
|
15
|
-
builder.add_production('E' => 'id')
|
|
16
|
-
builder
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
# Basic tokenizing method
|
|
20
|
-
def tokenize(aText, aGrammar)
|
|
21
|
-
tokens = aText.scan(/\S+/).map do |lexeme|
|
|
22
|
-
case lexeme
|
|
23
|
-
when '+'
|
|
24
|
-
terminal = aGrammar.name2symbol[lexeme]
|
|
25
|
-
when /^[_a-zA-Z][_a-zA-Z0-9]*$/
|
|
26
|
-
terminal = aGrammar.name2symbol['id']
|
|
27
|
-
else
|
|
28
|
-
msg = "Unknown input text '#{lexeme}'"
|
|
29
|
-
fail StandardError, msg
|
|
30
|
-
end
|
|
31
|
-
Rley::Parser::Token.new(lexeme, terminal)
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
return tokens
|
|
35
|
-
end
|
|
36
|
-
end # module
|
|
1
|
+
# Load the builder class
|
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
|
3
|
+
require_relative '../../../lib/rley/parser/token'
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
module AmbiguousGrammarHelper
|
|
7
|
+
# Factory method. Creates a grammar builder for a basic ambiguous
|
|
8
|
+
# expression grammar.
|
|
9
|
+
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
|
10
|
+
def grammar_builder()
|
|
11
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
|
12
|
+
builder.add_terminals('+', 'id')
|
|
13
|
+
builder.add_production('S' => 'E')
|
|
14
|
+
builder.add_production('E' => %w(E + E))
|
|
15
|
+
builder.add_production('E' => 'id')
|
|
16
|
+
builder
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Basic tokenizing method
|
|
20
|
+
def tokenize(aText, aGrammar)
|
|
21
|
+
tokens = aText.scan(/\S+/).map do |lexeme|
|
|
22
|
+
case lexeme
|
|
23
|
+
when '+'
|
|
24
|
+
terminal = aGrammar.name2symbol[lexeme]
|
|
25
|
+
when /^[_a-zA-Z][_a-zA-Z0-9]*$/
|
|
26
|
+
terminal = aGrammar.name2symbol['id']
|
|
27
|
+
else
|
|
28
|
+
msg = "Unknown input text '#{lexeme}'"
|
|
29
|
+
fail StandardError, msg
|
|
30
|
+
end
|
|
31
|
+
Rley::Parser::Token.new(lexeme, terminal)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
return tokens
|
|
35
|
+
end
|
|
36
|
+
end # module
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Load the builder class
|
|
2
|
+
require_relative '../../../lib/rley/parser/token'
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
# Mixin module implementing expectation helper methods.
|
|
6
|
+
module ExpectationHelper
|
|
7
|
+
# Helper method. Compare the data from all the parse entries
|
|
8
|
+
# of a given ParseEntrySet with an array of expectation strings.
|
|
9
|
+
def compare_entry_texts(anEntrySet, expectations)
|
|
10
|
+
(0...expectations.size).each do |i|
|
|
11
|
+
expect(anEntrySet.entries[i].to_s).to eq(expectations[i])
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Helper method. Compare the antecedents from all the parse entries
|
|
16
|
+
# of a given ParseEntrySet at given position with a Hash of the form:
|
|
17
|
+
# consequent label => [ antecedent label(s) ]
|
|
18
|
+
def check_antecedence(aParsing, aPosition, expectations)
|
|
19
|
+
entry_set = aParsing.chart[aPosition]
|
|
20
|
+
|
|
21
|
+
expectations.each do |consequent_label, antecedent_labels|
|
|
22
|
+
consequent = entry_set.entries.find do |entry|
|
|
23
|
+
entry.to_s == consequent_label
|
|
24
|
+
end
|
|
25
|
+
actual_antecedents = aParsing.antecedence.fetch(consequent)
|
|
26
|
+
expect(actual_antecedents.map(&:to_s)).to eq(antecedent_labels)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def expected_terminals(anEntrySet, termNames)
|
|
32
|
+
terminals = anEntrySet.expected_terminals
|
|
33
|
+
actual_names = terminals.map(&:name)
|
|
34
|
+
expect(actual_names.sort).to eq(termNames.sort)
|
|
35
|
+
end
|
|
36
|
+
end # module
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Load the builder class
|
|
2
|
+
require_relative '../../../lib/rley/parser/token'
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
# Mixin module implementing helper methods.
|
|
6
|
+
module GrammarHelper
|
|
7
|
+
# Create a sequence of tokens, one for each grammar symbol name.
|
|
8
|
+
# Synopsis:
|
|
9
|
+
# build_token_sequence(%w(a a b c c), grm1)
|
|
10
|
+
def build_token_sequence(literals, aGrammar)
|
|
11
|
+
tokens = literals.map do |lexeme|
|
|
12
|
+
case lexeme
|
|
13
|
+
when String
|
|
14
|
+
terminal = aGrammar.name2symbol[lexeme]
|
|
15
|
+
Rley::Parser::Token.new(lexeme, terminal)
|
|
16
|
+
|
|
17
|
+
when Hash # lexeme is reality a Hash: literal => terminal name
|
|
18
|
+
sub_array = lexeme.to_a
|
|
19
|
+
sub_array.map do |(literal, name)|
|
|
20
|
+
terminal = aGrammar.name2symbol[name]
|
|
21
|
+
Rley::Parser::Token.new(literal, terminal)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
return tokens.flatten
|
|
27
|
+
end
|
|
28
|
+
end # module
|