rley 0.2.01 → 0.2.02
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/CHANGELOG.md +5 -0
- data/README.md +9 -6
- data/examples/parsers/parsing_L0.rb +1 -1
- data/examples/parsers/parsing_L1.rb +136 -0
- data/examples/parsers/parsing_abc.rb +1 -2
- data/examples/parsers/parsing_ambig.rb +85 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/formatter/json.rb +1 -4
- data/lib/rley/parser/parse_state_tracker.rb +1 -1
- data/lib/rley/parser/parse_tree_builder.rb +12 -13
- data/lib/rley/parser/parsing.rb +61 -44
- data/lib/rley/ptree/non_terminal_node.rb +13 -0
- data/lib/rley/ptree/parse_tree_node.rb +7 -0
- data/lib/rley/ptree/terminal_node.rb +7 -0
- data/lib/rley/ptree/token_range.rb +18 -0
- data/spec/rley/formatter/json_spec.rb +1 -2
- data/spec/rley/parser/earley_parser_spec.rb +77 -4
- data/spec/rley/parser/parse_tree_builder_spec.rb +28 -27
- data/spec/rley/parser/parsing_spec.rb +230 -73
- data/spec/rley/ptree/non_terminal_node_spec.rb +42 -3
- data/spec/rley/ptree/terminal_node_spec.rb +39 -0
- data/spec/rley/ptree/token_range_spec.rb +45 -0
- data/spec/rley/support/ambiguous_grammar_helper.rb +36 -0
- data/spec/rley/support/grammar_b_expr_helper.rb +1 -1
- metadata +7 -2
@@ -1,13 +1,22 @@
|
|
1
|
+
require 'ostruct'
|
1
2
|
require_relative '../../spec_helper'
|
2
3
|
|
4
|
+
require_relative '../../../lib/rley/ptree/terminal_node'
|
3
5
|
# Load the class under test
|
4
6
|
require_relative '../../../lib/rley/ptree/non_terminal_node'
|
5
7
|
|
6
8
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
7
9
|
module PTree # Open this namespace to avoid module qualifier prefixes
|
8
10
|
describe NonTerminalNode do
|
9
|
-
|
10
|
-
|
11
|
+
# Factory method. Generate a range from its boundary values.
|
12
|
+
def range(low, high)
|
13
|
+
return TokenRange.new(low: low, high: high)
|
14
|
+
end
|
15
|
+
|
16
|
+
let(:sample_symbol) do
|
17
|
+
OpenStruct.new(name: 'VP')
|
18
|
+
end
|
19
|
+
let(:sample_range) { range(0, 3) }
|
11
20
|
|
12
21
|
subject { NonTerminalNode.new(sample_symbol, sample_range) }
|
13
22
|
|
@@ -18,7 +27,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
18
27
|
end # context
|
19
28
|
|
20
29
|
context 'Provided services:' do
|
21
|
-
it 'should accept children' do
|
30
|
+
it 'should accept the addition of children' do
|
22
31
|
child1 = double('first_child')
|
23
32
|
child2 = double('second_child')
|
24
33
|
child3 = double('third_child')
|
@@ -27,6 +36,36 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
27
36
|
subject.add_child(child3)
|
28
37
|
expect(subject.children).to eq([child1, child2, child3])
|
29
38
|
end
|
39
|
+
|
40
|
+
it 'should provide a text representation of itself' do
|
41
|
+
# Case 1: no child
|
42
|
+
expected_text = "VP[0, 3]"
|
43
|
+
expect(subject.to_string(0)).to eq(expected_text)
|
44
|
+
|
45
|
+
# Case 2: with children
|
46
|
+
child_1_1 = TerminalNode.new(OpenStruct.new(name: 'Verb'), range(0, 1))
|
47
|
+
child_1_2 = NonTerminalNode.new(OpenStruct.new(name: 'NP'), range(1, 3))
|
48
|
+
child_2_1 = TerminalNode.new(OpenStruct.new(name: 'Determiner'), range(1, 2))
|
49
|
+
child_2_2 = NonTerminalNode.new(OpenStruct.new(name: 'Nominal'), range(2, 3))
|
50
|
+
child_3_1 = TerminalNode.new(OpenStruct.new(name: 'Noun'), range(2, 3))
|
51
|
+
subject.add_child(child_1_1)
|
52
|
+
subject.add_child(child_1_2)
|
53
|
+
child_1_2.add_child(child_2_1)
|
54
|
+
child_1_2.add_child(child_2_2)
|
55
|
+
child_2_2.add_child(child_3_1)
|
56
|
+
child_1_1.token = OpenStruct.new(lexeme: 'catch')
|
57
|
+
child_2_1.token = OpenStruct.new(lexeme: 'that')
|
58
|
+
child_3_1.token = OpenStruct.new(lexeme: 'bus')
|
59
|
+
expected_text = <<-SNIPPET
|
60
|
+
VP[0, 3]
|
61
|
+
+- Verb[0, 1]: 'catch'
|
62
|
+
+- NP[1, 3]
|
63
|
+
+- Determiner[1, 2]: 'that'
|
64
|
+
+- Nominal[2, 3]
|
65
|
+
+- Noun[2, 3]: 'bus'
|
66
|
+
SNIPPET
|
67
|
+
expect(subject.to_string(0)).to eq(expected_text.chomp)
|
68
|
+
end
|
30
69
|
end # context
|
31
70
|
end # describe
|
32
71
|
end # module
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
require_relative '../../spec_helper'
|
3
|
+
|
4
|
+
# Load the class under test
|
5
|
+
require_relative '../../../lib/rley/ptree/terminal_node'
|
6
|
+
|
7
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
8
|
+
module PTree # Open this namespace to avoid module qualifier prefixes
|
9
|
+
describe TerminalNode do
|
10
|
+
let(:sample_symbol) do
|
11
|
+
OpenStruct.new(name: 'Noun')
|
12
|
+
end
|
13
|
+
let(:sample_range) { double('fake-range') }
|
14
|
+
|
15
|
+
subject { TerminalNode.new(sample_symbol, sample_range) }
|
16
|
+
|
17
|
+
context 'Initialization:' do
|
18
|
+
it "shouldn't be already bound to a token" do
|
19
|
+
expect(subject.token).to be_nil
|
20
|
+
end
|
21
|
+
end # context
|
22
|
+
|
23
|
+
context 'Provided services:' do
|
24
|
+
it 'should provide a text representation of itself' do
|
25
|
+
# Case 1: not bound to a token
|
26
|
+
expected_text = "Noun[?, ?]: '(nil)'"
|
27
|
+
expect(subject.to_string(0)).to eq(expected_text)
|
28
|
+
|
29
|
+
# Case 2: bound to token
|
30
|
+
subject.token = OpenStruct.new(lexeme: 'peace')
|
31
|
+
expected_text = "Noun[?, ?]: 'peace'"
|
32
|
+
expect(subject.to_string(0)).to eq(expected_text)
|
33
|
+
end
|
34
|
+
end # context
|
35
|
+
end # describe
|
36
|
+
end # module
|
37
|
+
end # module
|
38
|
+
|
39
|
+
# End of file
|
@@ -116,6 +116,51 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
116
116
|
expect(instance.low).to eq(1)
|
117
117
|
expect(instance.high).to eq(4)
|
118
118
|
end
|
119
|
+
|
120
|
+
it 'should tell whether an index value lies outside the range' do
|
121
|
+
# Out of range...
|
122
|
+
expect(subject.out_of_range?(-1)).to eq(true)
|
123
|
+
expect(subject.out_of_range?(6)).to eq(true)
|
124
|
+
|
125
|
+
# On boundaries...
|
126
|
+
expect(subject.out_of_range?(0)).to eq(false)
|
127
|
+
expect(subject.out_of_range?(5)).to eq(false)
|
128
|
+
|
129
|
+
# Inside boundaries
|
130
|
+
expect(subject.out_of_range?(2)).to eq(false)
|
131
|
+
|
132
|
+
instance = TokenRange.new(low: nil, high: 5)
|
133
|
+
|
134
|
+
# Lower bound is nil
|
135
|
+
expect(instance.out_of_range?(-1)).to eq(false)
|
136
|
+
expect(instance.out_of_range?(5)).to eq(false)
|
137
|
+
expect(instance.out_of_range?(6)).to eq(true)
|
138
|
+
|
139
|
+
instance = TokenRange.new(low: 0, high: nil)
|
140
|
+
|
141
|
+
# Upper bound is nil
|
142
|
+
expect(instance.out_of_range?(-1)).to eq(true)
|
143
|
+
expect(instance.out_of_range?(0)).to eq(false)
|
144
|
+
expect(instance.out_of_range?(6)).to eq(false)
|
145
|
+
end
|
146
|
+
|
147
|
+
it 'should provide a text representation of itself' do
|
148
|
+
# Case 1: not bound is set
|
149
|
+
instance = TokenRange.new({})
|
150
|
+
expect(instance.to_string(0)).to eq('[?, ?]')
|
151
|
+
|
152
|
+
# Case: only low bound is set
|
153
|
+
instance = TokenRange.new(low: 0)
|
154
|
+
expect(instance.to_string(0)).to eq('[0, ?]')
|
155
|
+
|
156
|
+
# Case: only upper bound is set
|
157
|
+
instance = TokenRange.new(high: 5)
|
158
|
+
expect(instance.to_string(0)).to eq('[?, 5]')
|
159
|
+
|
160
|
+
# Case: both bounds are set
|
161
|
+
instance = TokenRange.new(low: 0, high: 5)
|
162
|
+
expect(instance.to_string(0)).to eq('[0, 5]')
|
163
|
+
end
|
119
164
|
end
|
120
165
|
end # describe
|
121
166
|
end # module
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# Load the builder class
|
2
|
+
require_relative '../../../lib/rley/syntax/grammar_builder'
|
3
|
+
require_relative '../../../lib/rley/parser/token'
|
4
|
+
|
5
|
+
|
6
|
+
module AmbiguousGrammarHelper
|
7
|
+
# Factory method. Creates a grammar builder for a basic ambiguous
|
8
|
+
# expression grammar.
|
9
|
+
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
10
|
+
def grammar_builder()
|
11
|
+
builder = Rley::Syntax::GrammarBuilder.new
|
12
|
+
builder.add_terminals('+', 'id')
|
13
|
+
builder.add_production('S' => 'E')
|
14
|
+
builder.add_production('E' => %w(E + E))
|
15
|
+
builder.add_production('E' => 'id')
|
16
|
+
builder
|
17
|
+
end
|
18
|
+
|
19
|
+
# Basic tokenizing method
|
20
|
+
def tokenize(aText, aGrammar)
|
21
|
+
tokens = aText.scan(/\S+/).map do |lexeme|
|
22
|
+
case lexeme
|
23
|
+
when '+'
|
24
|
+
terminal = aGrammar.name2symbol[lexeme]
|
25
|
+
when /^[_a-zA-Z][_a-zA-Z0-9]*$/
|
26
|
+
terminal = aGrammar.name2symbol['id']
|
27
|
+
else
|
28
|
+
msg = "Unknown input text '#{lexeme}'"
|
29
|
+
fail StandardError, msg
|
30
|
+
end
|
31
|
+
Rley::Parser::Token.new(lexeme, terminal)
|
32
|
+
end
|
33
|
+
|
34
|
+
return tokens
|
35
|
+
end
|
36
|
+
end # module
|
@@ -6,7 +6,7 @@ require_relative '../../../lib/rley/parser/token'
|
|
6
6
|
module GrammarBExprHelper
|
7
7
|
# Factory method. Creates a grammar builder for a basic arithmetic
|
8
8
|
# expression grammar.
|
9
|
-
# (based on
|
9
|
+
# (based on the article about Earley's algorithm in Wikipedia)
|
10
10
|
def grammar_expr_builder()
|
11
11
|
builder = Rley::Syntax::GrammarBuilder.new
|
12
12
|
builder.add_terminals('+', '*', 'integer')
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rley
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.02
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dimitri Geshef
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-02-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -102,8 +102,10 @@ files:
|
|
102
102
|
- examples/grammars/grammar_abc.rb
|
103
103
|
- examples/grammars/grammar_L0.rb
|
104
104
|
- examples/parsers/parsing_abc.rb
|
105
|
+
- examples/parsers/parsing_ambig.rb
|
105
106
|
- examples/parsers/parsing_b_expr.rb
|
106
107
|
- examples/parsers/parsing_L0.rb
|
108
|
+
- examples/parsers/parsing_L1.rb
|
107
109
|
- examples/recognizers/recognizer_abc.rb
|
108
110
|
- lib/rley.rb
|
109
111
|
- lib/rley/constants.rb
|
@@ -148,7 +150,9 @@ files:
|
|
148
150
|
- spec/rley/ptree/non_terminal_node_spec.rb
|
149
151
|
- spec/rley/ptree/parse_tree_node_spec.rb
|
150
152
|
- spec/rley/ptree/parse_tree_spec.rb
|
153
|
+
- spec/rley/ptree/terminal_node_spec.rb
|
151
154
|
- spec/rley/ptree/token_range_spec.rb
|
155
|
+
- spec/rley/support/ambiguous_grammar_helper.rb
|
152
156
|
- spec/rley/support/grammar_abc_helper.rb
|
153
157
|
- spec/rley/support/grammar_b_expr_helper.rb
|
154
158
|
- spec/rley/syntax/grammar_builder_spec.rb
|
@@ -207,6 +211,7 @@ test_files:
|
|
207
211
|
- spec/rley/ptree/non_terminal_node_spec.rb
|
208
212
|
- spec/rley/ptree/parse_tree_node_spec.rb
|
209
213
|
- spec/rley/ptree/parse_tree_spec.rb
|
214
|
+
- spec/rley/ptree/terminal_node_spec.rb
|
210
215
|
- spec/rley/ptree/token_range_spec.rb
|
211
216
|
- spec/rley/syntax/grammar_builder_spec.rb
|
212
217
|
- spec/rley/syntax/grammar_spec.rb
|