dendroid 0.2.00 → 0.2.02
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -0
- data/CHANGELOG.md +16 -0
- data/lib/dendroid/formatters/ascii_tree.rb +142 -142
- data/lib/dendroid/formatters/base_formatter.rb +24 -25
- data/lib/dendroid/formatters/bracket_notation.rb +50 -50
- data/lib/dendroid/grm_analysis/dotted_item.rb +0 -1
- data/lib/dendroid/grm_analysis/grm_analyzer.rb +9 -1
- data/lib/dendroid/grm_analysis/rule_items.rb +1 -1
- data/lib/dendroid/parsing/and_node.rb +54 -56
- data/lib/dendroid/parsing/chart_walker.rb +301 -293
- data/lib/dendroid/parsing/composite_parse_node.rb +20 -21
- data/lib/dendroid/parsing/empty_rule_node.rb +28 -28
- data/lib/dendroid/parsing/or_node.rb +46 -51
- data/lib/dendroid/parsing/parse_node.rb +26 -26
- data/lib/dendroid/parsing/parse_tree_visitor.rb +127 -127
- data/lib/dendroid/parsing/terminal_node.rb +32 -32
- data/lib/dendroid/parsing/walk_progress.rb +121 -117
- data/lib/dendroid/recognizer/chart.rb +1 -6
- data/lib/dendroid/recognizer/recognizer.rb +12 -6
- data/lib/dendroid/syntax/grammar.rb +3 -5
- data/lib/dendroid/syntax/rule.rb +1 -1
- data/lib/dendroid.rb +13 -0
- data/spec/dendroid/grm_analysis/dotted_item_spec.rb +1 -1
- data/spec/dendroid/parsing/chart_walker_spec.rb +250 -223
- data/spec/dendroid/parsing/terminal_node_spec.rb +36 -36
- data/spec/dendroid/recognizer/recognizer_spec.rb +0 -1
- data/spec/dendroid/support/sample_grammars.rb +0 -2
- data/version.txt +1 -1
- metadata +3 -4
- data/lib/dendroid/parsing/parser.rb +0 -185
@@ -1,185 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'weakref'
|
4
|
-
require_relative '../grm_dsl/base_grm_builder'
|
5
|
-
require_relative '../utils/base_tokenizer'
|
6
|
-
require_relative '../recognizer/recognizer'
|
7
|
-
require_relative 'chart_walker'
|
8
|
-
require_relative 'parse_tree_visitor'
|
9
|
-
require_relative '../formatters/bracket_notation'
|
10
|
-
require_relative '../formatters/ascii_tree'
|
11
|
-
|
12
|
-
module Dendroid
|
13
|
-
module Parsing
|
14
|
-
class Parser
|
15
|
-
end # class
|
16
|
-
end # module
|
17
|
-
end # module
|
18
|
-
|
19
|
-
def grammar_l1
|
20
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
21
|
-
# Grammar inspired from Wikipedia entry on Earley parsing
|
22
|
-
declare_terminals('PLUS', 'STAR', 'INTEGER')
|
23
|
-
rule('p' => 's')
|
24
|
-
rule('s' => ['s PLUS m', 'm'])
|
25
|
-
rule('m' => ['m STAR t', 't'])
|
26
|
-
rule('t' => 'INTEGER')
|
27
|
-
end
|
28
|
-
|
29
|
-
builder.grammar
|
30
|
-
end
|
31
|
-
|
32
|
-
def grammar_l31
|
33
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
34
|
-
# Ambiguous arithmetical expression language
|
35
|
-
# This language is compatible with tokenizer L1
|
36
|
-
declare_terminals('PLUS', 'STAR', 'INTEGER')
|
37
|
-
rule('p' => 's')
|
38
|
-
rule('s' => ['s PLUS s', 's STAR s', 'INTEGER'])
|
39
|
-
end
|
40
|
-
|
41
|
-
builder.grammar
|
42
|
-
end
|
43
|
-
|
44
|
-
def tokenizer_l1
|
45
|
-
Dendroid::Utils::BaseTokenizer.new do
|
46
|
-
map_verbatim2terminal({ '+' => :PLUS, '*' => :STAR })
|
47
|
-
|
48
|
-
scan_verbatim(['+', '*'])
|
49
|
-
scan_value(/\d+/, :INTEGER, ->(txt) { txt.to_i })
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def retrieve_success_item(chart, grammar)
|
54
|
-
last_item_set = chart.item_sets.last
|
55
|
-
result = nil
|
56
|
-
last_item_set.items.reverse_each do |itm|
|
57
|
-
if itm.origin.zero? && itm.dotted_item.completed? && itm.dotted_item.rule.lhs == grammar.start_symbol
|
58
|
-
result = itm
|
59
|
-
break
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
result
|
64
|
-
end
|
65
|
-
|
66
|
-
def grammar_l10
|
67
|
-
# Grammar with left recursive rule
|
68
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
69
|
-
declare_terminals('a')
|
70
|
-
|
71
|
-
rule 'A' => ['A a', '']
|
72
|
-
end
|
73
|
-
|
74
|
-
builder.grammar
|
75
|
-
end
|
76
|
-
|
77
|
-
def tokenizer_l10
|
78
|
-
Dendroid::Utils::BaseTokenizer.new do
|
79
|
-
map_verbatim2terminal({ 'a' => :a })
|
80
|
-
|
81
|
-
scan_verbatim(['a'])
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
def grammar_l11
|
86
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
87
|
-
# Grammar with right-recursive rule
|
88
|
-
declare_terminals('a')
|
89
|
-
|
90
|
-
rule 'A' => ['a A', '']
|
91
|
-
end
|
92
|
-
|
93
|
-
builder.grammar
|
94
|
-
end
|
95
|
-
|
96
|
-
def tokenizer_l11
|
97
|
-
Dendroid::Utils::BaseTokenizer.new do
|
98
|
-
map_verbatim2terminal({ 'a' => :a })
|
99
|
-
|
100
|
-
scan_verbatim(['a'])
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def grammar_l8
|
105
|
-
builder = Dendroid::GrmDSL::BaseGrmBuilder.new do
|
106
|
-
# (based on grammar G2 from paper Masaru Tomita "An Efficient Context-Free Parsing Algorithm
|
107
|
-
# for Natural Languages")
|
108
|
-
declare_terminals('x')
|
109
|
-
|
110
|
-
rule 'S' => ['S S', 'x']
|
111
|
-
end
|
112
|
-
|
113
|
-
builder.grammar
|
114
|
-
end
|
115
|
-
|
116
|
-
def tokenizer_l8
|
117
|
-
Dendroid::Utils::BaseTokenizer.new do
|
118
|
-
map_verbatim2terminal({ 'x' => :x })
|
119
|
-
|
120
|
-
scan_verbatim(['x'])
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
########################################
|
125
|
-
# Entry point
|
126
|
-
########################################
|
127
|
-
kode = 3
|
128
|
-
|
129
|
-
case kode
|
130
|
-
when 0
|
131
|
-
recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l1, tokenizer_l1)
|
132
|
-
chart = recognizer.run('2 + 3 * 4')
|
133
|
-
succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
|
134
|
-
walker = Dendroid::Parsing::ChartWalker.new(chart)
|
135
|
-
root = walker.walk(succ_item)
|
136
|
-
|
137
|
-
# formatter = BracketNotation.new($stdout)
|
138
|
-
formatter = Asciitree.new($stdout)
|
139
|
-
visitor = ParseTreeVisitor.new(root)
|
140
|
-
formatter.render(visitor)
|
141
|
-
|
142
|
-
when 1
|
143
|
-
recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l10, tokenizer_l10)
|
144
|
-
chart = recognizer.run('a a a a a')
|
145
|
-
succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
|
146
|
-
walker = Dendroid::Parsing::ChartWalker.new(chart)
|
147
|
-
root = walker.walk(succ_item)
|
148
|
-
|
149
|
-
formatter = Asciitree.new($stdout)
|
150
|
-
visitor = ParseTreeVisitor.new(root)
|
151
|
-
formatter.render(visitor)
|
152
|
-
|
153
|
-
when 2
|
154
|
-
recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l11, tokenizer_l11)
|
155
|
-
chart = recognizer.run('a a a a a')
|
156
|
-
succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
|
157
|
-
walker = Dendroid::Parsing::ChartWalker.new(chart)
|
158
|
-
root = walker.walk(succ_item)
|
159
|
-
|
160
|
-
formatter = Asciitree.new($stdout)
|
161
|
-
visitor = ParseTreeVisitor.new(root)
|
162
|
-
formatter.render(visitor)
|
163
|
-
|
164
|
-
when 3
|
165
|
-
recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l31, tokenizer_l1)
|
166
|
-
chart = recognizer.run('2 + 3 * 4')
|
167
|
-
succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
|
168
|
-
walker = Dendroid::Parsing::ChartWalker.new(chart)
|
169
|
-
root = walker.walk(succ_item)
|
170
|
-
|
171
|
-
formatter = Asciitree.new($stdout)
|
172
|
-
visitor = ParseTreeVisitor.new(root)
|
173
|
-
formatter.render(visitor)
|
174
|
-
|
175
|
-
when 4
|
176
|
-
recognizer = Dendroid::Recognizer::Recognizer.new(grammar_l8, tokenizer_l8)
|
177
|
-
chart = recognizer.run('x x x x')
|
178
|
-
succ_item = retrieve_success_item(chart, recognizer.grm_analysis.grammar)
|
179
|
-
walker = Dendroid::Parsing::ChartWalker.new(chart)
|
180
|
-
root = walker.walk(succ_item)
|
181
|
-
|
182
|
-
formatter = Asciitree.new($stdout)
|
183
|
-
visitor = ParseTreeVisitor.new(root)
|
184
|
-
formatter.render(visitor)
|
185
|
-
end
|