rley 0.7.03 → 0.7.08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +362 -62
- data/.travis.yml +6 -7
- data/CHANGELOG.md +20 -1
- data/LICENSE.txt +1 -1
- data/README.md +6 -7
- data/Rakefile +2 -0
- data/appveyor.yml +2 -4
- data/examples/NLP/benchmark_pico_en.rb +2 -0
- data/examples/NLP/engtagger.rb +193 -188
- data/examples/NLP/nano_eng/nano_en_demo.rb +2 -0
- data/examples/NLP/nano_eng/nano_grammar.rb +7 -5
- data/examples/NLP/pico_en_demo.rb +2 -0
- data/examples/data_formats/JSON/cli_options.rb +3 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +14 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +14 -21
- data/examples/data_formats/JSON/json_demo.rb +2 -0
- data/examples/data_formats/JSON/json_grammar.rb +4 -2
- data/examples/data_formats/JSON/json_lexer.rb +10 -8
- data/examples/data_formats/JSON/json_minifier.rb +3 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +15 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +25 -37
- data/examples/general/calc_iter1/calc_demo.rb +2 -0
- data/examples/general/calc_iter1/calc_grammar.rb +4 -2
- data/examples/general/calc_iter1/calc_lexer.rb +8 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +7 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +7 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +29 -43
- data/examples/general/calc_iter2/calc_demo.rb +2 -0
- data/examples/general/calc_iter2/calc_grammar.rb +5 -3
- data/examples/general/calc_iter2/calc_lexer.rb +13 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +28 -26
- data/examples/general/left.rb +4 -2
- data/examples/general/right.rb +4 -2
- data/lib/rley.rb +2 -0
- data/lib/rley/base/base_parser.rb +2 -0
- data/lib/rley/base/dotted_item.rb +38 -41
- data/lib/rley/base/grm_items_builder.rb +2 -0
- data/lib/rley/constants.rb +5 -3
- data/lib/rley/engine.rb +22 -24
- data/lib/rley/formatter/asciitree.rb +6 -4
- data/lib/rley/formatter/base_formatter.rb +2 -0
- data/lib/rley/formatter/bracket_notation.rb +3 -8
- data/lib/rley/formatter/debug.rb +8 -6
- data/lib/rley/formatter/json.rb +4 -2
- data/lib/rley/gfg/call_edge.rb +3 -1
- data/lib/rley/gfg/edge.rb +7 -5
- data/lib/rley/gfg/end_vertex.rb +4 -6
- data/lib/rley/gfg/epsilon_edge.rb +3 -5
- data/lib/rley/gfg/grm_flow_graph.rb +31 -25
- data/lib/rley/gfg/item_vertex.rb +12 -22
- data/lib/rley/gfg/non_terminal_vertex.rb +6 -4
- data/lib/rley/gfg/return_edge.rb +2 -0
- data/lib/rley/gfg/scan_edge.rb +3 -1
- data/lib/rley/gfg/shortcut_edge.rb +4 -2
- data/lib/rley/gfg/start_vertex.rb +6 -8
- data/lib/rley/gfg/vertex.rb +47 -41
- data/lib/rley/lexical/token.rb +3 -1
- data/lib/rley/lexical/token_range.rb +8 -6
- data/lib/rley/parse_forest_visitor.rb +7 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +11 -11
- data/lib/rley/parse_rep/cst_builder.rb +7 -4
- data/lib/rley/parse_rep/parse_forest_builder.rb +36 -25
- data/lib/rley/parse_rep/parse_forest_factory.rb +5 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +18 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +15 -15
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -25
- data/lib/rley/parse_tree_visitor.rb +3 -1
- data/lib/rley/parser/error_reason.rb +9 -8
- data/lib/rley/parser/gfg_chart.rb +54 -22
- data/lib/rley/parser/gfg_earley_parser.rb +3 -1
- data/lib/rley/parser/gfg_parsing.rb +51 -31
- data/lib/rley/parser/parse_entry.rb +29 -33
- data/lib/rley/parser/parse_entry_set.rb +32 -27
- data/lib/rley/parser/parse_entry_tracker.rb +6 -4
- data/lib/rley/parser/parse_state.rb +18 -21
- data/lib/rley/parser/parse_state_tracker.rb +6 -4
- data/lib/rley/parser/parse_tracer.rb +15 -13
- data/lib/rley/parser/parse_walker_factory.rb +28 -29
- data/lib/rley/parser/state_set.rb +11 -10
- data/lib/rley/ptree/non_terminal_node.rb +10 -6
- data/lib/rley/ptree/parse_tree.rb +6 -4
- data/lib/rley/ptree/parse_tree_node.rb +7 -5
- data/lib/rley/ptree/terminal_node.rb +9 -7
- data/lib/rley/rley_error.rb +12 -10
- data/lib/rley/sppf/alternative_node.rb +8 -6
- data/lib/rley/sppf/composite_node.rb +9 -7
- data/lib/rley/sppf/epsilon_node.rb +5 -3
- data/lib/rley/sppf/leaf_node.rb +5 -3
- data/lib/rley/sppf/non_terminal_node.rb +2 -0
- data/lib/rley/sppf/parse_forest.rb +19 -17
- data/lib/rley/sppf/sppf_node.rb +9 -8
- data/lib/rley/sppf/token_node.rb +5 -3
- data/lib/rley/syntax/grammar.rb +7 -5
- data/lib/rley/syntax/grammar_builder.rb +11 -9
- data/lib/rley/syntax/grm_symbol.rb +8 -6
- data/lib/rley/syntax/literal.rb +2 -0
- data/lib/rley/syntax/non_terminal.rb +11 -15
- data/lib/rley/syntax/production.rb +13 -11
- data/lib/rley/syntax/symbol_seq.rb +10 -10
- data/lib/rley/syntax/terminal.rb +6 -5
- data/lib/rley/syntax/verbatim_symbol.rb +5 -3
- data/lib/support/base_tokenizer.rb +23 -20
- data/spec/rley/base/dotted_item_spec.rb +4 -2
- data/spec/rley/base/grm_items_builder_spec.rb +2 -0
- data/spec/rley/engine_spec.rb +47 -9
- data/spec/rley/formatter/asciitree_spec.rb +11 -9
- data/spec/rley/formatter/bracket_notation_spec.rb +16 -14
- data/spec/rley/formatter/debug_spec.rb +4 -2
- data/spec/rley/formatter/json_spec.rb +5 -3
- data/spec/rley/gfg/call_edge_spec.rb +2 -0
- data/spec/rley/gfg/edge_spec.rb +2 -0
- data/spec/rley/gfg/end_vertex_spec.rb +7 -5
- data/spec/rley/gfg/epsilon_edge_spec.rb +2 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -0
- data/spec/rley/gfg/item_vertex_spec.rb +12 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +5 -3
- data/spec/rley/gfg/return_edge_spec.rb +2 -0
- data/spec/rley/gfg/scan_edge_spec.rb +2 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +3 -1
- data/spec/rley/gfg/start_vertex_spec.rb +7 -5
- data/spec/rley/gfg/vertex_spec.rb +5 -3
- data/spec/rley/lexical/token_range_spec.rb +18 -16
- data/spec/rley/lexical/token_spec.rb +4 -2
- data/spec/rley/parse_forest_visitor_spec.rb +167 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +46 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +8 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +7 -5
- data/spec/rley/parse_rep/groucho_spec.rb +25 -25
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +28 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +8 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +4 -2
- data/spec/rley/parse_tree_visitor_spec.rb +12 -8
- data/spec/rley/parser/error_reason_spec.rb +8 -6
- data/spec/rley/parser/gfg_chart_spec.rb +17 -4
- data/spec/rley/parser/gfg_earley_parser_spec.rb +16 -11
- data/spec/rley/parser/gfg_parsing_spec.rb +41 -252
- data/spec/rley/parser/parse_entry_set_spec.rb +2 -0
- data/spec/rley/parser/parse_entry_spec.rb +21 -19
- data/spec/rley/parser/parse_state_spec.rb +7 -5
- data/spec/rley/parser/parse_tracer_spec.rb +16 -14
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -8
- data/spec/rley/parser/state_set_spec.rb +24 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +7 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +6 -4
- data/spec/rley/ptree/parse_tree_spec.rb +2 -0
- data/spec/rley/ptree/terminal_node_spec.rb +8 -6
- data/spec/rley/sppf/alternative_node_spec.rb +8 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +5 -3
- data/spec/rley/sppf/token_node_spec.rb +6 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +5 -4
- data/spec/rley/support/expectation_helper.rb +2 -0
- data/spec/rley/support/grammar_abc_helper.rb +4 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +6 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +6 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +6 -5
- data/spec/rley/support/grammar_helper.rb +2 -0
- data/spec/rley/support/grammar_l0_helper.rb +15 -16
- data/spec/rley/support/grammar_pb_helper.rb +8 -5
- data/spec/rley/support/grammar_sppf_helper.rb +3 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +7 -5
- data/spec/rley/syntax/grammar_spec.rb +8 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +3 -1
- data/spec/rley/syntax/literal_spec.rb +2 -0
- data/spec/rley/syntax/non_terminal_spec.rb +10 -8
- data/spec/rley/syntax/production_spec.rb +15 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +4 -2
- data/spec/rley/syntax/terminal_spec.rb +7 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +3 -1
- data/spec/spec_helper.rb +2 -12
- data/spec/support/base_tokenizer_spec.rb +9 -2
- metadata +21 -63
- data/.simplecov +0 -7
- data/Gemfile +0 -8
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Grammar for a simple subset of English language
|
2
4
|
# It is called nano-English because it has a more elaborate
|
3
5
|
# grammar than pico-English but remains still tiny compared to "real" English
|
@@ -9,7 +11,7 @@ require 'rley' # Load the gem
|
|
9
11
|
# based on chapter 12 from Jurafski & Martin book.
|
10
12
|
# Daniel Jurafsky, James H. Martin: "Speech and Language Processing";
|
11
13
|
# 2009, Pearson Education, Inc., ISBN 978-0135041963
|
12
|
-
# It defines the syntax of a sentence in a mini English-like language
|
14
|
+
# It defines the syntax of a sentence in a mini English-like language
|
13
15
|
builder = Rley::Syntax::GrammarBuilder.new do
|
14
16
|
add_terminals('Pronoun', 'Proper-Noun')
|
15
17
|
add_terminals('Determiner', 'Noun')
|
@@ -19,7 +21,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
19
21
|
|
20
22
|
rule 'language' => 'sentence'
|
21
23
|
rule 'sentence' => 'declarative'
|
22
|
-
rule 'sentence' => 'imperative'
|
24
|
+
rule 'sentence' => 'imperative'
|
23
25
|
rule 'sentence' => 'yes_no_question'
|
24
26
|
rule 'sentence' => 'wh_subject_question'
|
25
27
|
rule 'sentence' => 'wh_non_subject_question'
|
@@ -31,7 +33,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
31
33
|
rule 'NP' => %w[Predeterminer NP]
|
32
34
|
rule 'NP' => 'Pronoun'
|
33
35
|
rule 'NP' => 'Proper-Noun'
|
34
|
-
rule 'NP' => %w[Det Card Ord Quant Nominal]
|
36
|
+
rule 'NP' => %w[Det Card Ord Quant Nominal]
|
35
37
|
rule 'VP' => 'Verb'
|
36
38
|
rule 'VP' => %w[Verb NP]
|
37
39
|
rule 'VP' => %w[Verb NP PP]
|
@@ -41,12 +43,12 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
41
43
|
rule 'Card' => 'Cardinal_number'
|
42
44
|
rule 'Card' => []
|
43
45
|
rule 'Ord' => 'Ordinal_number'
|
44
|
-
rule 'Ord' => []
|
46
|
+
rule 'Ord' => []
|
45
47
|
rule 'Nominal' => 'Noun'
|
46
48
|
rule 'Nominal' => %w[Nominal Noun]
|
47
49
|
rule 'Nominal' => %w[Nominal GerundVP]
|
48
50
|
rule 'Nominal' => %w[Nominal RelClause]
|
49
|
-
rule 'PP' => %w[Preposition NP]
|
51
|
+
rule 'PP' => %w[Preposition NP]
|
50
52
|
rule 'GerundVP' => 'GerundV'
|
51
53
|
rule 'GerundVP' => %w[GerundV NP]
|
52
54
|
rule 'GerundVP' => %w[GerundV NP PP]
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'optparse'
|
2
4
|
|
3
5
|
# A Hash specialization that collects the command-line options
|
@@ -53,7 +55,7 @@ END_TEXT
|
|
53
55
|
|
54
56
|
rep_help = <<-END_TEXT
|
55
57
|
Set the parse tree representation (default: cst)
|
56
|
-
cst Concrete Syntax Tree. The out-of-the-box parse tree
|
58
|
+
cst Concrete Syntax Tree. The out-of-the-box parse tree
|
57
59
|
representation.
|
58
60
|
ast Abstract Syntaxt Tree. A customized parse tree for JSON.
|
59
61
|
It is a more compact and practical representation.
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'json_ast_nodes'
|
2
4
|
|
3
5
|
# The purpose of a JSONASTBuilder is to build piece by piece an AST
|
@@ -7,7 +9,7 @@ require_relative 'json_ast_nodes'
|
|
7
9
|
# The Builder pattern creates a complex object
|
8
10
|
# (say, a parse tree) from simpler objects (terminal and non-terminal
|
9
11
|
# nodes) and using a step by step approach.
|
10
|
-
class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
12
|
+
class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
11
13
|
Terminal2NodeClass = {
|
12
14
|
'false' => JSONBooleanNode,
|
13
15
|
'true' => JSONBooleanNode,
|
@@ -17,21 +19,23 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
17
19
|
}.freeze
|
18
20
|
|
19
21
|
protected
|
20
|
-
|
21
|
-
def terminal2node
|
22
|
+
|
23
|
+
def terminal2node
|
22
24
|
Terminal2NodeClass
|
23
25
|
end
|
24
|
-
|
26
|
+
|
25
27
|
# Default class for representing terminal nodes.
|
26
28
|
# @return [Class]
|
27
|
-
def terminalnode_class
|
29
|
+
def terminalnode_class
|
28
30
|
JSONTerminalNode
|
29
31
|
end
|
30
|
-
|
32
|
+
|
33
|
+
# rubocop: disable Naming/VariableNumber
|
34
|
+
|
31
35
|
def reduce_JSON_text_0(_aProd, aRange, theTokens, theChildren)
|
32
36
|
return_first_child(aRange, theTokens, theChildren)
|
33
37
|
end
|
34
|
-
|
38
|
+
|
35
39
|
# rule 'object' => %w[begin-object member-list end-object]
|
36
40
|
def reduce_object_0(aProduction, _range, _tokens, theChildren)
|
37
41
|
second_child = theChildren[1]
|
@@ -67,7 +71,7 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
67
71
|
def reduce_array_0(aProduction, _range, _tokens, theChildren)
|
68
72
|
second_child = theChildren[1]
|
69
73
|
second_child.symbol = aProduction.lhs
|
70
|
-
return second_child
|
74
|
+
return second_child
|
71
75
|
end
|
72
76
|
|
73
77
|
# rule 'array' => %w[begin-array end-array]
|
@@ -81,12 +85,13 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
81
85
|
node.children << theChildren[2]
|
82
86
|
return node
|
83
87
|
end
|
84
|
-
|
88
|
+
|
85
89
|
# rule 'array-items' => %w[value]
|
86
90
|
def reduce_array_items_1(aProduction, _range, _tokens, theChildren)
|
87
91
|
node = JSONArrayNode.new(aProduction.lhs)
|
88
92
|
node.children << theChildren[0]
|
89
93
|
return node
|
90
94
|
end
|
95
|
+
# rubocop: enable Naming/VariableNumber
|
91
96
|
end # class
|
92
97
|
# End of file
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Classes that implement nodes of Abstract Syntax Trees (AST) representing
|
2
4
|
# JSON parse results.
|
3
5
|
|
@@ -14,11 +16,11 @@ JSONTerminalNode = Struct.new(:token, :value, :position) do
|
|
14
16
|
self.value = aLiteral.dup
|
15
17
|
end
|
16
18
|
|
17
|
-
def symbol
|
19
|
+
def symbol
|
18
20
|
token.terminal
|
19
21
|
end
|
20
22
|
|
21
|
-
def to_ruby
|
23
|
+
def to_ruby
|
22
24
|
value
|
23
25
|
end
|
24
26
|
|
@@ -27,9 +29,9 @@ JSONTerminalNode = Struct.new(:token, :value, :position) do
|
|
27
29
|
def accept(aVisitor)
|
28
30
|
aVisitor.visit_terminal(self)
|
29
31
|
end
|
30
|
-
|
32
|
+
|
31
33
|
def done!
|
32
|
-
# Do nothing
|
34
|
+
# Do nothing
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
@@ -75,23 +77,18 @@ class JSONCompositeNode
|
|
75
77
|
def accept(aVisitor)
|
76
78
|
aVisitor.visit_nonterminal(self)
|
77
79
|
end
|
78
|
-
|
80
|
+
|
79
81
|
def done!
|
80
|
-
# Do nothing
|
82
|
+
# Do nothing
|
81
83
|
end
|
82
84
|
|
83
85
|
alias subnodes children
|
84
86
|
end # class
|
85
87
|
|
86
|
-
|
87
88
|
class JSONArrayNode < JSONCompositeNode
|
88
|
-
def initialize(aSymbol)
|
89
|
-
super(aSymbol)
|
90
|
-
end
|
91
|
-
|
92
89
|
# Convert this tree node in a simpler Ruby representation.
|
93
90
|
# Basically a JSON object corresponds to a Ruhy Hash
|
94
|
-
def to_ruby
|
91
|
+
def to_ruby
|
95
92
|
rep = []
|
96
93
|
children.each do |child|
|
97
94
|
rep << child.to_ruby
|
@@ -112,7 +109,7 @@ class JSONPair
|
|
112
109
|
@symbol = aSymbol
|
113
110
|
end
|
114
111
|
|
115
|
-
def children
|
112
|
+
def children
|
116
113
|
return [name, value]
|
117
114
|
end
|
118
115
|
|
@@ -123,27 +120,23 @@ class JSONPair
|
|
123
120
|
def accept(aVisitor)
|
124
121
|
aVisitor.visit_nonterminal(self)
|
125
122
|
end
|
126
|
-
|
123
|
+
|
127
124
|
def done!
|
128
125
|
# Do nothing
|
129
126
|
end
|
130
|
-
|
127
|
+
|
131
128
|
def to_ruby
|
132
129
|
rep = {}
|
133
130
|
rep[name.to_ruby] = value.to_ruby
|
134
131
|
|
135
|
-
return rep
|
132
|
+
return rep
|
136
133
|
end
|
137
134
|
end # class
|
138
135
|
|
139
136
|
class JSONObjectNode < JSONCompositeNode
|
140
|
-
def initialize(aSymbol)
|
141
|
-
super(aSymbol)
|
142
|
-
end
|
143
|
-
|
144
137
|
# Convert this tree node in a simpler Ruby representation.
|
145
138
|
# Basically a JSON object corresponds to a Ruby Hash
|
146
|
-
def to_ruby
|
139
|
+
def to_ruby
|
147
140
|
rep = {}
|
148
141
|
members.each do |pair|
|
149
142
|
rep[pair.name.to_ruby] = pair.value.to_ruby
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Grammar for JSON data representation
|
2
4
|
require 'rley' # Load the gem
|
3
5
|
|
@@ -16,11 +18,11 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
16
18
|
rule 'JSON_text' => 'value'
|
17
19
|
rule 'value' => 'false'
|
18
20
|
rule 'value' => 'null'
|
19
|
-
rule 'value' => 'true'
|
21
|
+
rule 'value' => 'true'
|
20
22
|
rule 'value' => 'object'
|
21
23
|
rule 'value' => 'array'
|
22
24
|
rule 'value' => 'number'
|
23
|
-
rule 'value' => 'string'
|
25
|
+
rule 'value' => 'string'
|
24
26
|
rule 'object' => %w[begin-object member-list end-object]
|
25
27
|
rule 'object' => %w[begin-object end-object]
|
26
28
|
# Next rule is an example of a left recursive rule
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# File: json_lexer.rb
|
2
4
|
# Lexer for the JSON data format
|
3
5
|
require 'rley' # Load the gem
|
@@ -26,19 +28,19 @@ class JSONLexer
|
|
26
28
|
@line_start = 0
|
27
29
|
end
|
28
30
|
|
29
|
-
def tokens
|
31
|
+
def tokens
|
30
32
|
tok_sequence = []
|
31
33
|
until @scanner.eos?
|
32
34
|
token = _next_token
|
33
35
|
tok_sequence << token unless token.nil?
|
34
36
|
end
|
35
37
|
|
36
|
-
|
38
|
+
tok_sequence
|
37
39
|
end
|
38
40
|
|
39
41
|
private
|
40
42
|
|
41
|
-
def _next_token
|
43
|
+
def _next_token
|
42
44
|
token = nil
|
43
45
|
skip_whitespaces
|
44
46
|
curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
|
@@ -56,7 +58,7 @@ class JSONLexer
|
|
56
58
|
keyw = scanner.scan(/false|true|null/)
|
57
59
|
if keyw.nil?
|
58
60
|
invalid_keyw = scanner.scan(/\w+/)
|
59
|
-
raise ScanError
|
61
|
+
raise ScanError, "Invalid keyword: #{invalid_keyw}"
|
60
62
|
else
|
61
63
|
token = build_token(keyw, keyw)
|
62
64
|
end
|
@@ -66,7 +68,7 @@ class JSONLexer
|
|
66
68
|
value = scanner.scan(/([^"\\]|\\.)*/)
|
67
69
|
end_delimiter = scanner.getch
|
68
70
|
err_msg = 'No closing quotes (") found'
|
69
|
-
raise ScanError
|
71
|
+
raise ScanError, err_msg if end_delimiter.nil?
|
70
72
|
|
71
73
|
token = build_token(value, 'string')
|
72
74
|
|
@@ -79,12 +81,12 @@ class JSONLexer
|
|
79
81
|
erroneous = curr_ch.nil? ? '' : curr_ch
|
80
82
|
sequel = scanner.scan(/.{1,20}/)
|
81
83
|
erroneous += sequel unless sequel.nil?
|
82
|
-
raise ScanError
|
84
|
+
raise ScanError, "Unknown token #{erroneous}"
|
83
85
|
end # case
|
84
86
|
break unless token.nil? && (curr_ch = scanner.getch)
|
85
87
|
end
|
86
88
|
|
87
|
-
|
89
|
+
token
|
88
90
|
end
|
89
91
|
|
90
92
|
def build_token(lexeme, token)
|
@@ -92,7 +94,7 @@ class JSONLexer
|
|
92
94
|
Rley::Lexical::Token.new(lexeme, token, pos)
|
93
95
|
end
|
94
96
|
|
95
|
-
def skip_whitespaces
|
97
|
+
def skip_whitespaces
|
96
98
|
matched = scanner.scan(/[ \t\f\n\r]+/)
|
97
99
|
return if matched.nil?
|
98
100
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# File: json_minifier.rb
|
2
4
|
|
3
5
|
|
@@ -33,7 +35,7 @@ class JSONMinifier
|
|
33
35
|
lexeme = aTerm.token.lexeme
|
34
36
|
literal = if aTerm.symbol.name == 'string'
|
35
37
|
# String values are delimited by double quotes
|
36
|
-
|
38
|
+
"\"#{lexeme}\""
|
37
39
|
else
|
38
40
|
lexeme
|
39
41
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'calc_ast_nodes'
|
2
4
|
|
3
5
|
# The purpose of a CalcASTBuilder is to build piece by piece an AST
|
@@ -13,8 +15,8 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
13
15
|
}.freeze
|
14
16
|
|
15
17
|
protected
|
16
|
-
|
17
|
-
def terminal2node
|
18
|
+
|
19
|
+
def terminal2node
|
18
20
|
Terminal2NodeClass
|
19
21
|
end
|
20
22
|
|
@@ -24,7 +26,9 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
24
26
|
operator_node.children << theChildren[2]
|
25
27
|
return operator_node
|
26
28
|
end
|
27
|
-
|
29
|
+
|
30
|
+
# rubocop: disable Naming/VariableNumber
|
31
|
+
|
28
32
|
# rule 'expression' => 'simple_expression'
|
29
33
|
def reduce_expression_0(_production, aRange, theTokens, theChildren)
|
30
34
|
return_first_child(aRange, theTokens, theChildren)
|
@@ -33,32 +37,32 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
33
37
|
# rule 'simple_expression' => 'term'
|
34
38
|
def reduce_simple_expression_0(_production, aRange, theTokens, theChildren)
|
35
39
|
return_first_child(aRange, theTokens, theChildren)
|
36
|
-
end
|
40
|
+
end
|
37
41
|
|
38
42
|
# rule 'simple_expression' => %w[simple_expression add_operator term]
|
39
43
|
def reduce_simple_expression_1(_production, _range, _tokens, theChildren)
|
40
44
|
reduce_binary_operator(theChildren)
|
41
45
|
end
|
42
|
-
|
43
|
-
# rule 'term' => 'factor'
|
46
|
+
|
47
|
+
# rule 'term' => 'factor'
|
44
48
|
def reduce_term_0(_production, aRange, theTokens, theChildren)
|
45
49
|
return_first_child(aRange, theTokens, theChildren)
|
46
|
-
end
|
50
|
+
end
|
47
51
|
|
48
52
|
# rule 'term' => %w[term mul_operator factor]
|
49
53
|
def reduce_term_1(_production, _range, _tokens, theChildren)
|
50
54
|
reduce_binary_operator(theChildren)
|
51
55
|
end
|
52
|
-
|
56
|
+
|
53
57
|
# rule 'factor' => 'NUMBER'
|
54
58
|
def reduce_factor_0(_aProd, aRange, theTokens, theChildren)
|
55
59
|
return_first_child(aRange, theTokens, theChildren)
|
56
60
|
end
|
57
|
-
|
61
|
+
|
58
62
|
# # rule 'factor' => %w[LPAREN expression RPAREN]
|
59
63
|
def reduce_factor_1(_aProd, aRange, theTokens, theChildren)
|
60
64
|
return_second_child(aRange, theTokens, theChildren)
|
61
|
-
end
|
65
|
+
end
|
62
66
|
|
63
67
|
# rule 'add_operator' => 'PLUS'
|
64
68
|
def reduce_add_operator_0(_production, _range, _tokens, theChildren)
|
@@ -79,5 +83,6 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
79
83
|
def reduce_mul_operator_1(_production, _range, _tokens, theChildren)
|
80
84
|
return CalcDivideNode.new(theChildren[0].symbol)
|
81
85
|
end
|
86
|
+
# rubocop: enable Naming/VariableNumber
|
82
87
|
end # class
|
83
88
|
# End of file
|
@@ -1,7 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Classes that implement nodes of Abstract Syntax Trees (AST) representing
|
2
4
|
# calculator parse results.
|
3
5
|
|
4
|
-
|
5
6
|
CalcTerminalNode = Struct.new(:token, :value, :position) do
|
6
7
|
def initialize(aToken, aPosition)
|
7
8
|
self.token = aToken
|
@@ -14,23 +15,23 @@ CalcTerminalNode = Struct.new(:token, :value, :position) do
|
|
14
15
|
self.value = aLiteral.dup
|
15
16
|
end
|
16
17
|
|
17
|
-
def symbol
|
18
|
+
def symbol
|
18
19
|
token.terminal
|
19
20
|
end
|
20
21
|
|
21
|
-
def interpret
|
22
|
-
|
22
|
+
def interpret
|
23
|
+
value
|
23
24
|
end
|
24
|
-
|
25
|
-
def done!
|
25
|
+
|
26
|
+
def done!
|
26
27
|
# Do nothing
|
27
28
|
end
|
28
|
-
|
29
|
+
|
29
30
|
# Part of the 'visitee' role in Visitor design pattern.
|
30
31
|
# @param aVisitor[ParseTreeVisitor] the visitor
|
31
32
|
def accept(aVisitor)
|
32
33
|
aVisitor.visit_terminal(self)
|
33
|
-
end
|
34
|
+
end
|
34
35
|
end
|
35
36
|
|
36
37
|
class CalcNumberNode < CalcTerminalNode
|
@@ -51,30 +52,26 @@ class CalcCompositeNode
|
|
51
52
|
|
52
53
|
def initialize(aSymbol)
|
53
54
|
@symbol = aSymbol
|
54
|
-
@children = []
|
55
|
+
@children = []
|
55
56
|
end
|
56
|
-
|
57
|
-
def done!
|
57
|
+
|
58
|
+
def done!
|
58
59
|
# Do nothing
|
59
|
-
end
|
60
|
+
end
|
60
61
|
|
61
62
|
# Part of the 'visitee' role in Visitor design pattern.
|
62
63
|
# @param aVisitor[ParseTreeVisitor] the visitor
|
63
64
|
def accept(aVisitor)
|
64
65
|
aVisitor.visit_nonterminal(self)
|
65
66
|
end
|
66
|
-
|
67
|
+
|
67
68
|
alias subnodes children
|
68
69
|
end # class
|
69
70
|
|
70
71
|
class CalcUnaryOpNode < CalcCompositeNode
|
71
|
-
def initialize(aSymbol)
|
72
|
-
super(aSymbol)
|
73
|
-
end
|
74
|
-
|
75
72
|
# Convert this tree node in a simpler Ruby representation.
|
76
73
|
# Basically a Calc object corresponds to a Ruhy Hash
|
77
|
-
def to_ruby
|
74
|
+
def to_ruby
|
78
75
|
rep = {}
|
79
76
|
members.each do |pair|
|
80
77
|
rep[pair.name.to_ruby] = pair.value.to_ruby
|
@@ -90,13 +87,9 @@ class CalcNegateNode < CalcUnaryOpNode
|
|
90
87
|
end # class
|
91
88
|
|
92
89
|
class CalcBinaryOpNode < CalcCompositeNode
|
93
|
-
def initialize(aSymbol)
|
94
|
-
super(aSymbol)
|
95
|
-
end
|
96
|
-
|
97
90
|
protected
|
98
|
-
|
99
|
-
def retrieve_operands
|
91
|
+
|
92
|
+
def retrieve_operands
|
100
93
|
operands = []
|
101
94
|
children.each do |child|
|
102
95
|
oper = child.respond_to?(:interpret) ? child.interpret : child
|
@@ -109,42 +102,37 @@ end # class
|
|
109
102
|
|
110
103
|
class CalcAddNode < CalcBinaryOpNode
|
111
104
|
# TODO
|
112
|
-
def interpret
|
105
|
+
def interpret
|
113
106
|
operands = retrieve_operands
|
114
107
|
|
115
|
-
|
116
|
-
return sum
|
108
|
+
operands[0] + operands[1]
|
117
109
|
end
|
118
110
|
end # class
|
119
111
|
|
120
|
-
|
121
112
|
class CalcSubtractNode < CalcBinaryOpNode
|
122
113
|
# TODO
|
123
|
-
def interpret
|
114
|
+
def interpret
|
124
115
|
operands = retrieve_operands
|
125
116
|
|
126
|
-
|
127
|
-
return substraction
|
117
|
+
operands[0] - operands[1]
|
128
118
|
end
|
129
119
|
end # class
|
130
120
|
|
131
121
|
class CalcMultiplyNode < CalcBinaryOpNode
|
132
122
|
# TODO
|
133
|
-
def interpret
|
123
|
+
def interpret
|
134
124
|
operands = retrieve_operands
|
135
|
-
|
136
|
-
return multiplication
|
125
|
+
operands[0] * operands[1]
|
137
126
|
end
|
138
127
|
end # class
|
139
128
|
|
140
129
|
class CalcDivideNode < CalcBinaryOpNode
|
141
130
|
# TODO
|
142
|
-
def interpret
|
131
|
+
def interpret
|
143
132
|
operands = retrieve_operands
|
144
133
|
numerator = operands[0].to_f
|
145
134
|
denominator = operands[1]
|
146
|
-
|
147
|
-
return division
|
135
|
+
numerator / denominator
|
148
136
|
end
|
149
137
|
end # class
|
150
138
|
# End of file
|