dendroid 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -1
- data/CHANGELOG.md +16 -0
- data/dendroid.gemspec +2 -2
- data/lib/dendroid/grm_analysis/alternative_item.rb +1 -2
- data/lib/dendroid/grm_analysis/choice_items.rb +2 -1
- data/lib/dendroid/grm_analysis/dotted_item.rb +2 -0
- data/lib/dendroid/grm_analysis/grm_analyzer.rb +26 -9
- data/lib/dendroid/grm_analysis/production_items.rb +2 -1
- data/lib/dendroid/lexical/token.rb +1 -1
- data/lib/dendroid/lexical/token_position.rb +9 -1
- data/lib/dendroid/recognizer/chart.rb +53 -0
- data/lib/dendroid/recognizer/e_item.rb +48 -0
- data/lib/dendroid/recognizer/item_set.rb +37 -0
- data/lib/dendroid/recognizer/recognizer.rb +282 -0
- data/lib/dendroid/syntax/grm_symbol.rb +1 -1
- data/lib/dendroid/utils/base_tokenizer.rb +54 -15
- data/spec/dendroid/grm_analysis/grm_analyzer_spec.rb +7 -77
- data/spec/dendroid/lexical/token_position_spec.rb +1 -1
- data/spec/dendroid/recognizer/chart_spec.rb +2 -0
- data/spec/dendroid/recognizer/e_item_spec.rb +55 -0
- data/spec/dendroid/recognizer/item_set_spec.rb +63 -0
- data/spec/dendroid/recognizer/recognizer_spec.rb +186 -0
- data/spec/dendroid/support/sample_grammars.rb +76 -0
- data/spec/dendroid/utils/base_tokenizer_spec.rb +4 -4
- data/version.txt +1 -1
- metadata +13 -4
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Dendroid
|
4
|
-
#
|
4
|
+
# This module contains for all classes representing elements of .
|
5
5
|
module Syntax
|
6
6
|
# Abstract class for grammar symbols.
|
7
7
|
# A grammar symbol is an element that appears in grammar rules.
|
@@ -5,44 +5,67 @@ require_relative '../lexical/token_position'
|
|
5
5
|
require_relative '../lexical/literal'
|
6
6
|
|
7
7
|
module Dendroid
|
8
|
+
# This module contains helper classes (e.g. a tokenizer generator)
|
8
9
|
module Utils
|
10
|
+
# A basic tokenizer.
|
11
|
+
# Responsibility: break input into a sequence of token objects.
|
12
|
+
# This class defines a simple DSL to build a tokenizer.
|
9
13
|
class BaseTokenizer
|
14
|
+
# @return [StringScanner] Low-level input scanner
|
10
15
|
attr_reader :scanner
|
16
|
+
|
17
|
+
# @return [Integer] The current line number
|
11
18
|
attr_reader :lineno
|
19
|
+
|
20
|
+
# @return [Integer] Position of last start of line in the input string
|
12
21
|
attr_reader :line_start
|
22
|
+
|
23
|
+
# @return [Hash{Symbol, Array<Regexp>}]
|
13
24
|
attr_reader :actions
|
14
25
|
|
26
|
+
# Constructor
|
27
|
+
# @param aBlock [Proc]
|
15
28
|
def initialize(&aBlock)
|
16
29
|
@scanner = StringScanner.new('')
|
17
30
|
@actions = { skip: [], scan_verbatim: [], scan_value: [] }
|
18
31
|
defaults
|
32
|
+
return unless block_given?
|
19
33
|
|
20
|
-
|
21
|
-
instance_exec(&aBlock)
|
22
|
-
# grammar_complete!
|
23
|
-
end
|
34
|
+
instance_exec(&aBlock)
|
24
35
|
end
|
25
36
|
|
37
|
+
# Reset the tokenizer and set new text to tokenize
|
38
|
+
# @param source [String]
|
26
39
|
def input=(source)
|
27
|
-
scanner.string = source
|
28
40
|
reset
|
41
|
+
scanner.string = source
|
29
42
|
end
|
30
43
|
|
44
|
+
# Reset the tokenizer
|
31
45
|
def reset
|
32
46
|
@lineno = 1
|
33
47
|
@line_start = 0
|
48
|
+
scanner.reset
|
34
49
|
end
|
35
50
|
|
36
51
|
# action, pattern, terminal?, conversion?
|
37
52
|
# action = skip, skip_nl, scan
|
53
|
+
|
54
|
+
# Associate the provided pattern to the action of skipping a newline and
|
55
|
+
# incrementing the line counter.
|
56
|
+
# @param pattern [Regexp]
|
38
57
|
def skip_nl(pattern)
|
39
58
|
actions[:skip_nl] = pattern
|
40
59
|
end
|
41
60
|
|
61
|
+
# Associate the provided pattern with the action to skip whitespace(s).
|
62
|
+
# @param pattern [Regexp]
|
42
63
|
def skip_ws(pattern)
|
43
64
|
actions[:skip_ws] = pattern
|
44
65
|
end
|
45
66
|
|
67
|
+
# Associate the provided pattern with the action to skip the matching text.
|
68
|
+
# @param pattern [Regexp]
|
46
69
|
def skip(pattern)
|
47
70
|
if actions[:skip].empty?
|
48
71
|
actions[:skip] = pattern
|
@@ -52,6 +75,8 @@ module Dendroid
|
|
52
75
|
end
|
53
76
|
end
|
54
77
|
|
78
|
+
# Associate the provided pattern with the action to tokenize the matching text
|
79
|
+
# @param pattern [Regexp]
|
55
80
|
def scan_verbatim(pattern)
|
56
81
|
patt = normalize_pattern(pattern)
|
57
82
|
if actions[:scan_verbatim].empty?
|
@@ -62,9 +87,15 @@ module Dendroid
|
|
62
87
|
end
|
63
88
|
end
|
64
89
|
|
65
|
-
|
90
|
+
# Associate the provided pattern with the action to tokenize the matching text
|
91
|
+
# as an instance of the given terminal symbol and convert the matching text into
|
92
|
+
# a value by using the given conversion.
|
93
|
+
# @param pattern [Regexp]
|
94
|
+
# @param terminal [Dendroid::Syntax::Terminal]
|
95
|
+
# @param conversion [Proc] a Proc (lambda) that takes a String as argument and return a value.
|
96
|
+
def scan_value(pattern, terminal, conversion)
|
66
97
|
patt = normalize_pattern(pattern)
|
67
|
-
tuple = [patt, terminal,
|
98
|
+
tuple = [patt, terminal, conversion]
|
68
99
|
if actions[:scan_value].empty?
|
69
100
|
actions[:scan_value] = [tuple]
|
70
101
|
else
|
@@ -72,10 +103,16 @@ module Dendroid
|
|
72
103
|
end
|
73
104
|
end
|
74
105
|
|
106
|
+
# Set the mapping between a verbatim text to its corresponding terminal symbol name
|
107
|
+
# @param mapping [Hash{String, String}]
|
75
108
|
def map_verbatim2terminal(mapping)
|
76
109
|
@verbatim2terminal = mapping
|
77
110
|
end
|
78
111
|
|
112
|
+
# rubocop: disable Metrics/AbcSize
|
113
|
+
|
114
|
+
# Return the next token (if any) from the input stream.
|
115
|
+
# @return [Dendroid::Lexical::Token, NilClass]
|
79
116
|
def next_token
|
80
117
|
token = nil
|
81
118
|
|
@@ -93,7 +130,7 @@ module Dendroid
|
|
93
130
|
break
|
94
131
|
end
|
95
132
|
|
96
|
-
tuple = actions[:scan_value].find do |(pattern,
|
133
|
+
tuple = actions[:scan_value].find do |(pattern, _terminal, _conversion)|
|
97
134
|
scanner.check(pattern)
|
98
135
|
end
|
99
136
|
if tuple
|
@@ -106,18 +143,20 @@ module Dendroid
|
|
106
143
|
# Unknown token
|
107
144
|
col = scanner.pos - line_start + 1
|
108
145
|
erroneous = scanner.peek(1).nil? ? '' : scanner.scan(/./)
|
109
|
-
raise
|
146
|
+
raise StandardError, "Error: [line #{lineno}:#{col}]: Unexpected character #{erroneous}."
|
110
147
|
end
|
111
148
|
|
112
149
|
token
|
113
150
|
end
|
114
151
|
|
152
|
+
# rubocop: enable Metrics/AbcSize
|
153
|
+
|
115
154
|
protected
|
116
155
|
|
117
156
|
def defaults
|
118
157
|
# Defaults
|
119
|
-
skip_nl
|
120
|
-
skip_ws
|
158
|
+
skip_nl(/(?:\r\n)|\r|\n/) # Skip newlines
|
159
|
+
skip_ws(/[ \t\f]+/) # Skip blanks
|
121
160
|
end
|
122
161
|
|
123
162
|
private
|
@@ -146,7 +185,7 @@ module Dendroid
|
|
146
185
|
col = scanner.pos - lex_length - @line_start + 1
|
147
186
|
pos = Lexical::TokenPosition.new(@lineno, col)
|
148
187
|
token = Lexical::Token.new(text, pos, symbol_name)
|
149
|
-
rescue
|
188
|
+
rescue StandardError => e
|
150
189
|
puts "Failing with '#{symbol_name}' and '#{text}'"
|
151
190
|
raise e
|
152
191
|
end
|
@@ -154,15 +193,15 @@ module Dendroid
|
|
154
193
|
token
|
155
194
|
end
|
156
195
|
|
157
|
-
def value_scanned(aText, aSymbolName,
|
158
|
-
value =
|
196
|
+
def value_scanned(aText, aSymbolName, conversion)
|
197
|
+
value = conversion.call(aText)
|
159
198
|
lex_length = aText ? aText.size : 0
|
160
199
|
col = scanner.pos - lex_length - @line_start + 1
|
161
200
|
build_literal(aSymbolName, value, aText, col)
|
162
201
|
end
|
163
202
|
|
164
203
|
def build_literal(aSymbolName, aValue, aText, aPosition)
|
165
|
-
pos = if aPosition.
|
204
|
+
pos = if aPosition.is_a?(Integer)
|
166
205
|
col = aPosition
|
167
206
|
Lexical::TokenPosition.new(@lineno, col)
|
168
207
|
else
|
@@ -1,79 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative '../../spec_helper'
|
4
|
-
require_relative '
|
4
|
+
require_relative '../support/sample_grammars'
|
5
5
|
require_relative '../../../lib/dendroid/grm_analysis/grm_analyzer'
|
6
6
|
|
7
|
-
module SampleGrammars
|
8
|
-
def grammar_l1
|
9
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
10
|
-
# Grammar inspired from Wikipedia entry on Earley parsing
|
11
|
-
declare_terminals('PLUS', 'STAR', 'INTEGER')
|
12
|
-
|
13
|
-
rule('p' => 's')
|
14
|
-
rule('s' => ['s PLUS m', 'm'])
|
15
|
-
rule('m' => ['m STAR t', 't'])
|
16
|
-
rule('t' => 'INTEGER')
|
17
|
-
end
|
18
|
-
|
19
|
-
builder.grammar
|
20
|
-
end
|
21
|
-
|
22
|
-
def tokenizer_l1
|
23
|
-
Utils::BaseTokenizer.new do
|
24
|
-
map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
|
25
|
-
|
26
|
-
scan_verbatim(['+', '*'])
|
27
|
-
scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def grammar_l2
|
32
|
-
builder = GrmDSL::BaseGrmBuilder.new do
|
33
|
-
# Grammar inspired from Loup Vaillant's example
|
34
|
-
# https://loup-vaillant.fr/tutorials/earley-parsing/recogniser
|
35
|
-
declare_terminals('PLUS', 'MINUS', 'STAR', 'SLASH')
|
36
|
-
declare_terminals('LPAREN', 'RPAREN', 'NUMBER')
|
37
|
-
|
38
|
-
rule('p' => 'sum')
|
39
|
-
rule('sum' => ['sum PLUS product', 'sum MINUS product', 'product'])
|
40
|
-
rule('product' => ['product STAR factor', 'product SLASH factor', 'factor'])
|
41
|
-
rule('factor' => ['LPAREN sum RPAREN', 'NUMBER'])
|
42
|
-
end
|
43
|
-
|
44
|
-
builder.grammar
|
45
|
-
end
|
46
|
-
|
47
|
-
def tokenizer_l2
|
48
|
-
Utils::BaseTokenizer.new do
|
49
|
-
map_verbatim2terminal({
|
50
|
-
'+' => :PLUS,
|
51
|
-
'-' => :MINUS,
|
52
|
-
'*' => :STAR,
|
53
|
-
'/' => :SLASH,
|
54
|
-
'(' => :LPAREN,
|
55
|
-
')' => :RPAREN })
|
56
|
-
|
57
|
-
scan_verbatim(['+', '-', '*', '/', '(', ')'])
|
58
|
-
scan_value(/\d+/, :NUMBER, ->(txt) { txt.to_i })
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
|
-
def grammar_l3
|
63
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
64
|
-
# Grammar inspired from Andrew Appel's example
|
65
|
-
# Modern Compiler Implementation in Java
|
66
|
-
declare_terminals('a', 'c', 'd')
|
67
|
-
|
68
|
-
rule('Z' => ['d', 'X Y Z'])
|
69
|
-
rule('Y' => ['', 'c'])
|
70
|
-
rule('X' => ['Y', 'a'])
|
71
|
-
end
|
72
|
-
|
73
|
-
builder.grammar
|
74
|
-
end
|
75
|
-
end # module
|
76
|
-
|
77
7
|
describe Dendroid::GrmAnalysis::GrmAnalyzer do
|
78
8
|
include SampleGrammars
|
79
9
|
let(:grammar) { grammar_l1 }
|
@@ -91,7 +21,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
|
|
91
21
|
|
92
22
|
it 'knows the dotted items' do
|
93
23
|
item_count = subject.grammar.rules.reduce(0) do |count, prod|
|
94
|
-
count
|
24
|
+
count + prod.items.flatten.size
|
95
25
|
end
|
96
26
|
expect(subject.items.size).to eq(item_count)
|
97
27
|
expected_items = [
|
@@ -116,7 +46,7 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
|
|
116
46
|
end
|
117
47
|
|
118
48
|
it 'knows the item that follows a given dotted item' do
|
119
|
-
first_item = subject.items.find { |itm| itm.to_s ==
|
49
|
+
first_item = subject.items.find { |itm| itm.to_s == 'm => . m STAR t' }
|
120
50
|
second = subject.next_item(first_item)
|
121
51
|
expect(second.to_s).to eq('m => m . STAR t')
|
122
52
|
third = subject.next_item(second)
|
@@ -134,9 +64,9 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
|
|
134
64
|
'a' => ['a'],
|
135
65
|
'c' => ['c'],
|
136
66
|
'd' => ['d'],
|
137
|
-
'X' => [
|
67
|
+
'X' => %w[a c], # Add epsilon
|
138
68
|
'Y' => ['c'], # Add epsilon
|
139
|
-
'Z' => [
|
69
|
+
'Z' => %w[a c d]
|
140
70
|
}
|
141
71
|
expectations.each_pair do |sym_name, first_names|
|
142
72
|
symb = subject.grammar.name2symbol[sym_name]
|
@@ -149,8 +79,8 @@ describe Dendroid::GrmAnalysis::GrmAnalyzer do
|
|
149
79
|
it 'constructs the FOLLOW sets for non-terminal symbols' do
|
150
80
|
expectations = {
|
151
81
|
'Z' => [], # Add $$
|
152
|
-
'Y' => [
|
153
|
-
'X' => [
|
82
|
+
'Y' => %w[a c d],
|
83
|
+
'X' => %w[a c d]
|
154
84
|
}
|
155
85
|
subject.send(:build_follow_sets)
|
156
86
|
expectations.each_pair do |sym_name, follow_names|
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
require_relative '../../../lib/dendroid/syntax/terminal'
|
5
|
+
require_relative '../../../lib/dendroid/syntax/non_terminal'
|
6
|
+
require_relative '../../../lib/dendroid/syntax/symbol_seq'
|
7
|
+
require_relative '../../../lib/dendroid/syntax/production'
|
8
|
+
require_relative '../../../lib/dendroid/grm_analysis/dotted_item'
|
9
|
+
require_relative '../../../lib/dendroid/recognizer/e_item'
|
10
|
+
|
11
|
+
describe Dendroid::Recognizer::EItem do
|
12
|
+
let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
|
13
|
+
let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
|
14
|
+
let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
|
15
|
+
let(:rhs) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
|
16
|
+
let(:empty_body) { Dendroid::Syntax::SymbolSeq.new([]) }
|
17
|
+
let(:prod) { Dendroid::Syntax::Production.new(expr_symb, rhs) }
|
18
|
+
let(:empty_prod) { Dendroid::Syntax::Production.new(expr_symb, empty_body) }
|
19
|
+
let(:sample_dotted) { Dendroid::GrmAnalysis::DottedItem.new(prod, 1) }
|
20
|
+
let(:other_dotted) { Dendroid::GrmAnalysis::DottedItem.new(empty_prod, 0) }
|
21
|
+
let(:sample_origin) { 3 }
|
22
|
+
|
23
|
+
subject { described_class.new(sample_dotted, sample_origin) }
|
24
|
+
|
25
|
+
context 'Initialization:' do
|
26
|
+
it 'is initialized with a dotted item and an origin position' do
|
27
|
+
expect { described_class.new(sample_dotted, sample_origin) }.not_to raise_error
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'knows its related dotted item' do
|
31
|
+
expect(subject.dotted_item).to eq(sample_dotted)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'knows its origin value' do
|
35
|
+
expect(subject.origin).to eq(sample_origin)
|
36
|
+
end
|
37
|
+
end # context
|
38
|
+
|
39
|
+
context 'Provided service:' do
|
40
|
+
it 'knows the lhs of related production' do
|
41
|
+
expect(subject.lhs).to eq(expr_symb)
|
42
|
+
end # context
|
43
|
+
|
44
|
+
it 'can compare with another EItem' do
|
45
|
+
expect(subject == subject).to be_truthy
|
46
|
+
expect(subject == described_class.new(sample_dotted, sample_origin)).to be_truthy
|
47
|
+
expect(subject == described_class.new(sample_dotted, 2)).to be_falsey
|
48
|
+
expect(subject == described_class.new(other_dotted, sample_origin)).to be_falsey
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'can renders a String representation of itself' do
|
52
|
+
expect(subject.to_s).to eq("#{sample_dotted} @ #{sample_origin}")
|
53
|
+
end
|
54
|
+
end # context
|
55
|
+
end # describe
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
require_relative '../../../lib/dendroid/syntax/terminal'
|
5
|
+
require_relative '../../../lib/dendroid/syntax/non_terminal'
|
6
|
+
require_relative '../../../lib/dendroid/syntax/symbol_seq'
|
7
|
+
require_relative '../../../lib/dendroid/syntax/production'
|
8
|
+
require_relative '../../../lib/dendroid/grm_analysis/dotted_item'
|
9
|
+
require_relative '../../../lib/dendroid/recognizer/e_item'
|
10
|
+
require_relative '../../../lib/dendroid/recognizer/item_set'
|
11
|
+
|
12
|
+
describe Dendroid::Recognizer::ItemSet do
|
13
|
+
let(:num_symb) { Dendroid::Syntax::Terminal.new('NUMBER') }
|
14
|
+
let(:plus_symb) { Dendroid::Syntax::Terminal.new('PLUS') }
|
15
|
+
let(:expr_symb) { Dendroid::Syntax::NonTerminal.new('expression') }
|
16
|
+
let(:rhs) { Dendroid::Syntax::SymbolSeq.new([num_symb, plus_symb, num_symb]) }
|
17
|
+
let(:empty_body) { Dendroid::Syntax::SymbolSeq.new([]) }
|
18
|
+
let(:prod) { Dendroid::Syntax::Production.new(expr_symb, rhs) }
|
19
|
+
let(:empty_prod) { Dendroid::Syntax::Production.new(expr_symb, empty_body) }
|
20
|
+
let(:sample_dotted) { Dendroid::GrmAnalysis::DottedItem.new(prod, 1) }
|
21
|
+
let(:sample_origin) { 3 }
|
22
|
+
let(:other_dotted) { Dendroid::GrmAnalysis::DottedItem.new(empty_prod, 0) }
|
23
|
+
let(:first_element) { Dendroid::Recognizer::EItem.new(sample_dotted, sample_origin) }
|
24
|
+
let(:second_element) { Dendroid::Recognizer::EItem.new(other_dotted, 5) }
|
25
|
+
|
26
|
+
subject { described_class.new }
|
27
|
+
|
28
|
+
context 'Initialization:' do
|
29
|
+
it 'is initialized without argument' do
|
30
|
+
expect { described_class.new }.not_to raise_error
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'is empty at creation' do
|
34
|
+
expect(subject).to be_empty
|
35
|
+
end
|
36
|
+
end # context
|
37
|
+
|
38
|
+
context 'Provided services:' do
|
39
|
+
it 'adds a new element' do
|
40
|
+
subject.add_item(first_element)
|
41
|
+
expect(subject.size).to eq(1)
|
42
|
+
|
43
|
+
# Trying a second time, doesn't change the set
|
44
|
+
subject.add_item(first_element)
|
45
|
+
expect(subject.size).to eq(1)
|
46
|
+
|
47
|
+
subject.add_item(second_element)
|
48
|
+
expect(subject.size).to eq(2)
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'can render a String representation of itself' do
|
52
|
+
subject.add_item(first_element)
|
53
|
+
subject.add_item(second_element)
|
54
|
+
|
55
|
+
expectations = [
|
56
|
+
'expression => NUMBER . PLUS NUMBER @ 3',
|
57
|
+
'expression => . @ 5'
|
58
|
+
].join("\n")
|
59
|
+
|
60
|
+
expect(subject.to_s).to eq(expectations)
|
61
|
+
end
|
62
|
+
end # context
|
63
|
+
end # describe
|
@@ -0,0 +1,186 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
require_relative '../support/sample_grammars'
|
5
|
+
require_relative '../../../lib/dendroid/recognizer/recognizer'
|
6
|
+
|
7
|
+
describe Dendroid::Recognizer::Recognizer do
|
8
|
+
include SampleGrammars
|
9
|
+
let(:grammar1) { grammar_l1 }
|
10
|
+
|
11
|
+
# Implements a dotted item: expression => NUMBER . PLUS NUMBER
|
12
|
+
subject { described_class.new(grammar1, tokenizer_l1) }
|
13
|
+
|
14
|
+
context 'Initialization:' do
|
15
|
+
it 'is initialized with a grammar' do
|
16
|
+
expect { described_class.new(grammar1, tokenizer_l1) }.not_to raise_error
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'knows its grammar analyzer' do
|
20
|
+
expect(subject.grm_analysis).to be_kind_of(Dendroid::GrmAnalysis::GrmAnalyzer)
|
21
|
+
expect(subject.grm_analysis.grammar).to eq(grammar1)
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'knows its tokenizer' do
|
25
|
+
expect(subject.grm_analysis).to be_kind_of(Dendroid::GrmAnalysis::GrmAnalyzer)
|
26
|
+
expect(subject.grm_analysis.grammar).to eq(grammar1)
|
27
|
+
end
|
28
|
+
end # context
|
29
|
+
|
30
|
+
context 'Recognizer at work:' do
|
31
|
+
it 'can recognize example from Wikipedia' do
|
32
|
+
chart = subject.run('2 + 3 * 4')
|
33
|
+
expect(chart).to be_successful
|
34
|
+
|
35
|
+
set0 = [ # . 2 + 3 * 4'
|
36
|
+
'p => . s @ 0',
|
37
|
+
's => . s PLUS m @ 0',
|
38
|
+
's => . m @ 0',
|
39
|
+
'm => . m STAR t @ 0',
|
40
|
+
'm => . t @ 0',
|
41
|
+
't => . INTEGER @ 0'
|
42
|
+
]
|
43
|
+
set1 = [ # 2 . + 3 * 4'
|
44
|
+
't => INTEGER . @ 0',
|
45
|
+
'm => t . @ 0',
|
46
|
+
's => m . @ 0',
|
47
|
+
#'m => m . STAR t @ 0',
|
48
|
+
'p => s . @ 0',
|
49
|
+
's => s . PLUS m @ 0'
|
50
|
+
]
|
51
|
+
set2 = [ # 2 + . 3 * 4'
|
52
|
+
's => s PLUS . m @ 0',
|
53
|
+
'm => . m STAR t @ 2',
|
54
|
+
'm => . t @ 2',
|
55
|
+
't => . INTEGER @ 2'
|
56
|
+
]
|
57
|
+
set3 = [ # 2 + 3 . * 4'
|
58
|
+
't => INTEGER . @ 2',
|
59
|
+
'm => t . @ 2',
|
60
|
+
's => s PLUS m . @ 0',
|
61
|
+
'm => m . STAR t @ 2',
|
62
|
+
'p => s . @ 0',
|
63
|
+
# 's => s . PLUS m @ 0'
|
64
|
+
]
|
65
|
+
set4 = [ # 2 + 3 * . 4'
|
66
|
+
'm => m STAR . t @ 2',
|
67
|
+
't => . INTEGER @ 4'
|
68
|
+
]
|
69
|
+
set5 = [ # 2 + 3 * 4 .'
|
70
|
+
't => INTEGER . @ 4',
|
71
|
+
'm => m STAR t . @ 2',
|
72
|
+
's => s PLUS m . @ 0',
|
73
|
+
# 'm => m . STAR t @ 2',
|
74
|
+
'p => s . @ 0'
|
75
|
+
# 's => s . PLUS m @ 0'
|
76
|
+
]
|
77
|
+
[set0, set1, set2, set3, set4, set5].each_with_index do |set, rank|
|
78
|
+
expect(chart[rank].to_s).to eq(set.join("\n"))
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'can recognize example for L2 language' do
|
83
|
+
recognizer = described_class.new(grammar_l2, tokenizer_l2)
|
84
|
+
chart = recognizer.run('1 + (2 * 3 - 4)')
|
85
|
+
expect(chart).to be_successful
|
86
|
+
|
87
|
+
set0 = [ # . 1 + (2 * 3 - 4)
|
88
|
+
'p => . sum @ 0',
|
89
|
+
'sum => . sum PLUS product @ 0',
|
90
|
+
'sum => . sum MINUS product @ 0',
|
91
|
+
'sum => . product @ 0',
|
92
|
+
'product => . product STAR factor @ 0',
|
93
|
+
'product => . product SLASH factor @ 0',
|
94
|
+
'product => . factor @ 0',
|
95
|
+
# 'factor => . LPAREN sum RPAREN @ 0',
|
96
|
+
'factor => . NUMBER @ 0'
|
97
|
+
]
|
98
|
+
set1 = [ # 1 . + (2 * 3 - 4)
|
99
|
+
'factor => NUMBER . @ 0',
|
100
|
+
'product => factor . @ 0',
|
101
|
+
'sum => product . @ 0',
|
102
|
+
# 'product => product . STAR factor @ 0',
|
103
|
+
# 'product => product . SLASH factor @ 0',
|
104
|
+
'p => sum . @ 0',
|
105
|
+
'sum => sum . PLUS product @ 0',
|
106
|
+
# 'sum => sum . MINUS product @ 0'
|
107
|
+
]
|
108
|
+
set2 = [ # 1 + . (2 * 3 - 4)
|
109
|
+
'sum => sum PLUS . product @ 0',
|
110
|
+
'product => . product STAR factor @ 2',
|
111
|
+
'product => . product SLASH factor @ 2',
|
112
|
+
'product => . factor @ 2',
|
113
|
+
'factor => . LPAREN sum RPAREN @ 2',
|
114
|
+
# 'factor => . NUMBER @ 2'
|
115
|
+
]
|
116
|
+
set3 = [ # 1 + (. 2 * 3 - 4)
|
117
|
+
'factor => LPAREN . sum RPAREN @ 2',
|
118
|
+
'sum => . sum PLUS product @ 3',
|
119
|
+
'sum => . sum MINUS product @ 3',
|
120
|
+
'sum => . product @ 3',
|
121
|
+
'product => . product STAR factor @ 3',
|
122
|
+
'product => . product SLASH factor @ 3',
|
123
|
+
'product => . factor @ 3',
|
124
|
+
# 'factor => . LPAREN sum RPAREN @ 3',
|
125
|
+
'factor => . NUMBER @ 3'
|
126
|
+
]
|
127
|
+
set4 = [ # 1 + (2 . * 3 - 4)
|
128
|
+
'factor => NUMBER . @ 3',
|
129
|
+
'product => factor . @ 3',
|
130
|
+
'sum => product . @ 3',
|
131
|
+
'product => product . STAR factor @ 3',
|
132
|
+
# 'product => product . SLASH factor @ 3',
|
133
|
+
# 'factor => LPAREN sum . RPAREN @ 2',
|
134
|
+
# 'sum => sum . PLUS product @ 3',
|
135
|
+
# 'sum => sum . MINUS product @ 3'
|
136
|
+
]
|
137
|
+
set5 = [ # 1 + (2 * . 3 - 4)
|
138
|
+
'product => product STAR . factor @ 3',
|
139
|
+
# 'factor => . LPAREN sum RPAREN @ 5',
|
140
|
+
'factor => . NUMBER @ 5'
|
141
|
+
]
|
142
|
+
set6 = [ # 1 + (2 * 3 . - 4)
|
143
|
+
'factor => NUMBER . @ 5',
|
144
|
+
'product => product STAR factor . @ 3',
|
145
|
+
'sum => product . @ 3',
|
146
|
+
# 'product => product . STAR factor @ 3',
|
147
|
+
# 'product => product . SLASH factor @ 3',
|
148
|
+
# 'factor => LPAREN sum . RPAREN @ 2',
|
149
|
+
# 'sum => sum . PLUS product @ 3',
|
150
|
+
'sum => sum . MINUS product @ 3'
|
151
|
+
]
|
152
|
+
set7 = [ # 1 + (2 * 3 - . 4)
|
153
|
+
'sum => sum MINUS . product @ 3',
|
154
|
+
'product => . product STAR factor @ 7',
|
155
|
+
'product => . product SLASH factor @ 7',
|
156
|
+
'product => . factor @ 7',
|
157
|
+
# 'factor => . LPAREN sum RPAREN @ 7',
|
158
|
+
'factor => . NUMBER @ 7'
|
159
|
+
]
|
160
|
+
set8 = [ # 1 + (2 * 3 - 4 .)
|
161
|
+
'factor => NUMBER . @ 7',
|
162
|
+
'product => factor . @ 7',
|
163
|
+
'sum => sum MINUS product . @ 3',
|
164
|
+
# 'product => product . STAR factor @ 7',
|
165
|
+
# 'product => product . SLASH factor @ 7',
|
166
|
+
'factor => LPAREN sum . RPAREN @ 2',
|
167
|
+
# 'sum => sum . PLUS product @ 3',
|
168
|
+
# 'sum => sum . MINUS product @ 3'
|
169
|
+
]
|
170
|
+
set9 = [ # 1 + (2 * 3 - 4 ).
|
171
|
+
'factor => LPAREN sum RPAREN . @ 2',
|
172
|
+
'product => factor . @ 2',
|
173
|
+
'sum => sum PLUS product . @ 0',
|
174
|
+
# 'product => product . STAR factor @ 2',
|
175
|
+
# 'product => product . SLASH factor @ 2',
|
176
|
+
'p => sum . @ 0',
|
177
|
+
# 'sum => sum . PLUS product @ 0',
|
178
|
+
# 'sum => sum . MINUS product @ 0'
|
179
|
+
]
|
180
|
+
expectations = [set0, set1, set2, set3, set4, set5, set6, set7, set8, set9]
|
181
|
+
expectations.each_with_index do |set, rank|
|
182
|
+
expect(chart[rank].to_s).to eq(set.join("\n"))
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end # context
|
186
|
+
end # describe
|