rley 0.7.03 → 0.7.08
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +362 -62
- data/.travis.yml +6 -7
- data/CHANGELOG.md +20 -1
- data/LICENSE.txt +1 -1
- data/README.md +6 -7
- data/Rakefile +2 -0
- data/appveyor.yml +2 -4
- data/examples/NLP/benchmark_pico_en.rb +2 -0
- data/examples/NLP/engtagger.rb +193 -188
- data/examples/NLP/nano_eng/nano_en_demo.rb +2 -0
- data/examples/NLP/nano_eng/nano_grammar.rb +7 -5
- data/examples/NLP/pico_en_demo.rb +2 -0
- data/examples/data_formats/JSON/cli_options.rb +3 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +14 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +14 -21
- data/examples/data_formats/JSON/json_demo.rb +2 -0
- data/examples/data_formats/JSON/json_grammar.rb +4 -2
- data/examples/data_formats/JSON/json_lexer.rb +10 -8
- data/examples/data_formats/JSON/json_minifier.rb +3 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +15 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +25 -37
- data/examples/general/calc_iter1/calc_demo.rb +2 -0
- data/examples/general/calc_iter1/calc_grammar.rb +4 -2
- data/examples/general/calc_iter1/calc_lexer.rb +8 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +7 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +7 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +29 -43
- data/examples/general/calc_iter2/calc_demo.rb +2 -0
- data/examples/general/calc_iter2/calc_grammar.rb +5 -3
- data/examples/general/calc_iter2/calc_lexer.rb +13 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +28 -26
- data/examples/general/left.rb +4 -2
- data/examples/general/right.rb +4 -2
- data/lib/rley.rb +2 -0
- data/lib/rley/base/base_parser.rb +2 -0
- data/lib/rley/base/dotted_item.rb +38 -41
- data/lib/rley/base/grm_items_builder.rb +2 -0
- data/lib/rley/constants.rb +5 -3
- data/lib/rley/engine.rb +22 -24
- data/lib/rley/formatter/asciitree.rb +6 -4
- data/lib/rley/formatter/base_formatter.rb +2 -0
- data/lib/rley/formatter/bracket_notation.rb +3 -8
- data/lib/rley/formatter/debug.rb +8 -6
- data/lib/rley/formatter/json.rb +4 -2
- data/lib/rley/gfg/call_edge.rb +3 -1
- data/lib/rley/gfg/edge.rb +7 -5
- data/lib/rley/gfg/end_vertex.rb +4 -6
- data/lib/rley/gfg/epsilon_edge.rb +3 -5
- data/lib/rley/gfg/grm_flow_graph.rb +31 -25
- data/lib/rley/gfg/item_vertex.rb +12 -22
- data/lib/rley/gfg/non_terminal_vertex.rb +6 -4
- data/lib/rley/gfg/return_edge.rb +2 -0
- data/lib/rley/gfg/scan_edge.rb +3 -1
- data/lib/rley/gfg/shortcut_edge.rb +4 -2
- data/lib/rley/gfg/start_vertex.rb +6 -8
- data/lib/rley/gfg/vertex.rb +47 -41
- data/lib/rley/lexical/token.rb +3 -1
- data/lib/rley/lexical/token_range.rb +8 -6
- data/lib/rley/parse_forest_visitor.rb +7 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +11 -11
- data/lib/rley/parse_rep/cst_builder.rb +7 -4
- data/lib/rley/parse_rep/parse_forest_builder.rb +36 -25
- data/lib/rley/parse_rep/parse_forest_factory.rb +5 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +18 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +15 -15
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -25
- data/lib/rley/parse_tree_visitor.rb +3 -1
- data/lib/rley/parser/error_reason.rb +9 -8
- data/lib/rley/parser/gfg_chart.rb +54 -22
- data/lib/rley/parser/gfg_earley_parser.rb +3 -1
- data/lib/rley/parser/gfg_parsing.rb +51 -31
- data/lib/rley/parser/parse_entry.rb +29 -33
- data/lib/rley/parser/parse_entry_set.rb +32 -27
- data/lib/rley/parser/parse_entry_tracker.rb +6 -4
- data/lib/rley/parser/parse_state.rb +18 -21
- data/lib/rley/parser/parse_state_tracker.rb +6 -4
- data/lib/rley/parser/parse_tracer.rb +15 -13
- data/lib/rley/parser/parse_walker_factory.rb +28 -29
- data/lib/rley/parser/state_set.rb +11 -10
- data/lib/rley/ptree/non_terminal_node.rb +10 -6
- data/lib/rley/ptree/parse_tree.rb +6 -4
- data/lib/rley/ptree/parse_tree_node.rb +7 -5
- data/lib/rley/ptree/terminal_node.rb +9 -7
- data/lib/rley/rley_error.rb +12 -10
- data/lib/rley/sppf/alternative_node.rb +8 -6
- data/lib/rley/sppf/composite_node.rb +9 -7
- data/lib/rley/sppf/epsilon_node.rb +5 -3
- data/lib/rley/sppf/leaf_node.rb +5 -3
- data/lib/rley/sppf/non_terminal_node.rb +2 -0
- data/lib/rley/sppf/parse_forest.rb +19 -17
- data/lib/rley/sppf/sppf_node.rb +9 -8
- data/lib/rley/sppf/token_node.rb +5 -3
- data/lib/rley/syntax/grammar.rb +7 -5
- data/lib/rley/syntax/grammar_builder.rb +11 -9
- data/lib/rley/syntax/grm_symbol.rb +8 -6
- data/lib/rley/syntax/literal.rb +2 -0
- data/lib/rley/syntax/non_terminal.rb +11 -15
- data/lib/rley/syntax/production.rb +13 -11
- data/lib/rley/syntax/symbol_seq.rb +10 -10
- data/lib/rley/syntax/terminal.rb +6 -5
- data/lib/rley/syntax/verbatim_symbol.rb +5 -3
- data/lib/support/base_tokenizer.rb +23 -20
- data/spec/rley/base/dotted_item_spec.rb +4 -2
- data/spec/rley/base/grm_items_builder_spec.rb +2 -0
- data/spec/rley/engine_spec.rb +47 -9
- data/spec/rley/formatter/asciitree_spec.rb +11 -9
- data/spec/rley/formatter/bracket_notation_spec.rb +16 -14
- data/spec/rley/formatter/debug_spec.rb +4 -2
- data/spec/rley/formatter/json_spec.rb +5 -3
- data/spec/rley/gfg/call_edge_spec.rb +2 -0
- data/spec/rley/gfg/edge_spec.rb +2 -0
- data/spec/rley/gfg/end_vertex_spec.rb +7 -5
- data/spec/rley/gfg/epsilon_edge_spec.rb +2 -0
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -0
- data/spec/rley/gfg/item_vertex_spec.rb +12 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +5 -3
- data/spec/rley/gfg/return_edge_spec.rb +2 -0
- data/spec/rley/gfg/scan_edge_spec.rb +2 -0
- data/spec/rley/gfg/shortcut_edge_spec.rb +3 -1
- data/spec/rley/gfg/start_vertex_spec.rb +7 -5
- data/spec/rley/gfg/vertex_spec.rb +5 -3
- data/spec/rley/lexical/token_range_spec.rb +18 -16
- data/spec/rley/lexical/token_spec.rb +4 -2
- data/spec/rley/parse_forest_visitor_spec.rb +167 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +46 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +8 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +7 -5
- data/spec/rley/parse_rep/groucho_spec.rb +25 -25
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +28 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +8 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +4 -2
- data/spec/rley/parse_tree_visitor_spec.rb +12 -8
- data/spec/rley/parser/error_reason_spec.rb +8 -6
- data/spec/rley/parser/gfg_chart_spec.rb +17 -4
- data/spec/rley/parser/gfg_earley_parser_spec.rb +16 -11
- data/spec/rley/parser/gfg_parsing_spec.rb +41 -252
- data/spec/rley/parser/parse_entry_set_spec.rb +2 -0
- data/spec/rley/parser/parse_entry_spec.rb +21 -19
- data/spec/rley/parser/parse_state_spec.rb +7 -5
- data/spec/rley/parser/parse_tracer_spec.rb +16 -14
- data/spec/rley/parser/parse_walker_factory_spec.rb +10 -8
- data/spec/rley/parser/state_set_spec.rb +24 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +7 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +6 -4
- data/spec/rley/ptree/parse_tree_spec.rb +2 -0
- data/spec/rley/ptree/terminal_node_spec.rb +8 -6
- data/spec/rley/sppf/alternative_node_spec.rb +8 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +5 -3
- data/spec/rley/sppf/token_node_spec.rb +6 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +5 -4
- data/spec/rley/support/expectation_helper.rb +2 -0
- data/spec/rley/support/grammar_abc_helper.rb +4 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +6 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +6 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +6 -5
- data/spec/rley/support/grammar_helper.rb +2 -0
- data/spec/rley/support/grammar_l0_helper.rb +15 -16
- data/spec/rley/support/grammar_pb_helper.rb +8 -5
- data/spec/rley/support/grammar_sppf_helper.rb +3 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +7 -5
- data/spec/rley/syntax/grammar_spec.rb +8 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +3 -1
- data/spec/rley/syntax/literal_spec.rb +2 -0
- data/spec/rley/syntax/non_terminal_spec.rb +10 -8
- data/spec/rley/syntax/production_spec.rb +15 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +4 -2
- data/spec/rley/syntax/terminal_spec.rb +7 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +3 -1
- data/spec/spec_helper.rb +2 -12
- data/spec/support/base_tokenizer_spec.rb +9 -2
- metadata +21 -63
- data/.simplecov +0 -7
- data/Gemfile +0 -8
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Grammar for a simple subset of English language
|
2
4
|
# It is called nano-English because it has a more elaborate
|
3
5
|
# grammar than pico-English but remains still tiny compared to "real" English
|
@@ -9,7 +11,7 @@ require 'rley' # Load the gem
|
|
9
11
|
# based on chapter 12 from Jurafski & Martin book.
|
10
12
|
# Daniel Jurafsky, James H. Martin: "Speech and Language Processing";
|
11
13
|
# 2009, Pearson Education, Inc., ISBN 978-0135041963
|
12
|
-
# It defines the syntax of a sentence in a mini English-like language
|
14
|
+
# It defines the syntax of a sentence in a mini English-like language
|
13
15
|
builder = Rley::Syntax::GrammarBuilder.new do
|
14
16
|
add_terminals('Pronoun', 'Proper-Noun')
|
15
17
|
add_terminals('Determiner', 'Noun')
|
@@ -19,7 +21,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
19
21
|
|
20
22
|
rule 'language' => 'sentence'
|
21
23
|
rule 'sentence' => 'declarative'
|
22
|
-
rule 'sentence' => 'imperative'
|
24
|
+
rule 'sentence' => 'imperative'
|
23
25
|
rule 'sentence' => 'yes_no_question'
|
24
26
|
rule 'sentence' => 'wh_subject_question'
|
25
27
|
rule 'sentence' => 'wh_non_subject_question'
|
@@ -31,7 +33,7 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
31
33
|
rule 'NP' => %w[Predeterminer NP]
|
32
34
|
rule 'NP' => 'Pronoun'
|
33
35
|
rule 'NP' => 'Proper-Noun'
|
34
|
-
rule 'NP' => %w[Det Card Ord Quant Nominal]
|
36
|
+
rule 'NP' => %w[Det Card Ord Quant Nominal]
|
35
37
|
rule 'VP' => 'Verb'
|
36
38
|
rule 'VP' => %w[Verb NP]
|
37
39
|
rule 'VP' => %w[Verb NP PP]
|
@@ -41,12 +43,12 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
41
43
|
rule 'Card' => 'Cardinal_number'
|
42
44
|
rule 'Card' => []
|
43
45
|
rule 'Ord' => 'Ordinal_number'
|
44
|
-
rule 'Ord' => []
|
46
|
+
rule 'Ord' => []
|
45
47
|
rule 'Nominal' => 'Noun'
|
46
48
|
rule 'Nominal' => %w[Nominal Noun]
|
47
49
|
rule 'Nominal' => %w[Nominal GerundVP]
|
48
50
|
rule 'Nominal' => %w[Nominal RelClause]
|
49
|
-
rule 'PP' => %w[Preposition NP]
|
51
|
+
rule 'PP' => %w[Preposition NP]
|
50
52
|
rule 'GerundVP' => 'GerundV'
|
51
53
|
rule 'GerundVP' => %w[GerundV NP]
|
52
54
|
rule 'GerundVP' => %w[GerundV NP PP]
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'optparse'
|
2
4
|
|
3
5
|
# A Hash specialization that collects the command-line options
|
@@ -53,7 +55,7 @@ END_TEXT
|
|
53
55
|
|
54
56
|
rep_help = <<-END_TEXT
|
55
57
|
Set the parse tree representation (default: cst)
|
56
|
-
cst Concrete Syntax Tree. The out-of-the-box parse tree
|
58
|
+
cst Concrete Syntax Tree. The out-of-the-box parse tree
|
57
59
|
representation.
|
58
60
|
ast Abstract Syntaxt Tree. A customized parse tree for JSON.
|
59
61
|
It is a more compact and practical representation.
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'json_ast_nodes'
|
2
4
|
|
3
5
|
# The purpose of a JSONASTBuilder is to build piece by piece an AST
|
@@ -7,7 +9,7 @@ require_relative 'json_ast_nodes'
|
|
7
9
|
# The Builder pattern creates a complex object
|
8
10
|
# (say, a parse tree) from simpler objects (terminal and non-terminal
|
9
11
|
# nodes) and using a step by step approach.
|
10
|
-
class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
12
|
+
class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
11
13
|
Terminal2NodeClass = {
|
12
14
|
'false' => JSONBooleanNode,
|
13
15
|
'true' => JSONBooleanNode,
|
@@ -17,21 +19,23 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
17
19
|
}.freeze
|
18
20
|
|
19
21
|
protected
|
20
|
-
|
21
|
-
def terminal2node
|
22
|
+
|
23
|
+
def terminal2node
|
22
24
|
Terminal2NodeClass
|
23
25
|
end
|
24
|
-
|
26
|
+
|
25
27
|
# Default class for representing terminal nodes.
|
26
28
|
# @return [Class]
|
27
|
-
def terminalnode_class
|
29
|
+
def terminalnode_class
|
28
30
|
JSONTerminalNode
|
29
31
|
end
|
30
|
-
|
32
|
+
|
33
|
+
# rubocop: disable Naming/VariableNumber
|
34
|
+
|
31
35
|
def reduce_JSON_text_0(_aProd, aRange, theTokens, theChildren)
|
32
36
|
return_first_child(aRange, theTokens, theChildren)
|
33
37
|
end
|
34
|
-
|
38
|
+
|
35
39
|
# rule 'object' => %w[begin-object member-list end-object]
|
36
40
|
def reduce_object_0(aProduction, _range, _tokens, theChildren)
|
37
41
|
second_child = theChildren[1]
|
@@ -67,7 +71,7 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
67
71
|
def reduce_array_0(aProduction, _range, _tokens, theChildren)
|
68
72
|
second_child = theChildren[1]
|
69
73
|
second_child.symbol = aProduction.lhs
|
70
|
-
return second_child
|
74
|
+
return second_child
|
71
75
|
end
|
72
76
|
|
73
77
|
# rule 'array' => %w[begin-array end-array]
|
@@ -81,12 +85,13 @@ class JSONASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
81
85
|
node.children << theChildren[2]
|
82
86
|
return node
|
83
87
|
end
|
84
|
-
|
88
|
+
|
85
89
|
# rule 'array-items' => %w[value]
|
86
90
|
def reduce_array_items_1(aProduction, _range, _tokens, theChildren)
|
87
91
|
node = JSONArrayNode.new(aProduction.lhs)
|
88
92
|
node.children << theChildren[0]
|
89
93
|
return node
|
90
94
|
end
|
95
|
+
# rubocop: enable Naming/VariableNumber
|
91
96
|
end # class
|
92
97
|
# End of file
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Classes that implement nodes of Abstract Syntax Trees (AST) representing
|
2
4
|
# JSON parse results.
|
3
5
|
|
@@ -14,11 +16,11 @@ JSONTerminalNode = Struct.new(:token, :value, :position) do
|
|
14
16
|
self.value = aLiteral.dup
|
15
17
|
end
|
16
18
|
|
17
|
-
def symbol
|
19
|
+
def symbol
|
18
20
|
token.terminal
|
19
21
|
end
|
20
22
|
|
21
|
-
def to_ruby
|
23
|
+
def to_ruby
|
22
24
|
value
|
23
25
|
end
|
24
26
|
|
@@ -27,9 +29,9 @@ JSONTerminalNode = Struct.new(:token, :value, :position) do
|
|
27
29
|
def accept(aVisitor)
|
28
30
|
aVisitor.visit_terminal(self)
|
29
31
|
end
|
30
|
-
|
32
|
+
|
31
33
|
def done!
|
32
|
-
# Do nothing
|
34
|
+
# Do nothing
|
33
35
|
end
|
34
36
|
end
|
35
37
|
|
@@ -75,23 +77,18 @@ class JSONCompositeNode
|
|
75
77
|
def accept(aVisitor)
|
76
78
|
aVisitor.visit_nonterminal(self)
|
77
79
|
end
|
78
|
-
|
80
|
+
|
79
81
|
def done!
|
80
|
-
# Do nothing
|
82
|
+
# Do nothing
|
81
83
|
end
|
82
84
|
|
83
85
|
alias subnodes children
|
84
86
|
end # class
|
85
87
|
|
86
|
-
|
87
88
|
class JSONArrayNode < JSONCompositeNode
|
88
|
-
def initialize(aSymbol)
|
89
|
-
super(aSymbol)
|
90
|
-
end
|
91
|
-
|
92
89
|
# Convert this tree node in a simpler Ruby representation.
|
93
90
|
# Basically a JSON object corresponds to a Ruhy Hash
|
94
|
-
def to_ruby
|
91
|
+
def to_ruby
|
95
92
|
rep = []
|
96
93
|
children.each do |child|
|
97
94
|
rep << child.to_ruby
|
@@ -112,7 +109,7 @@ class JSONPair
|
|
112
109
|
@symbol = aSymbol
|
113
110
|
end
|
114
111
|
|
115
|
-
def children
|
112
|
+
def children
|
116
113
|
return [name, value]
|
117
114
|
end
|
118
115
|
|
@@ -123,27 +120,23 @@ class JSONPair
|
|
123
120
|
def accept(aVisitor)
|
124
121
|
aVisitor.visit_nonterminal(self)
|
125
122
|
end
|
126
|
-
|
123
|
+
|
127
124
|
def done!
|
128
125
|
# Do nothing
|
129
126
|
end
|
130
|
-
|
127
|
+
|
131
128
|
def to_ruby
|
132
129
|
rep = {}
|
133
130
|
rep[name.to_ruby] = value.to_ruby
|
134
131
|
|
135
|
-
return rep
|
132
|
+
return rep
|
136
133
|
end
|
137
134
|
end # class
|
138
135
|
|
139
136
|
class JSONObjectNode < JSONCompositeNode
|
140
|
-
def initialize(aSymbol)
|
141
|
-
super(aSymbol)
|
142
|
-
end
|
143
|
-
|
144
137
|
# Convert this tree node in a simpler Ruby representation.
|
145
138
|
# Basically a JSON object corresponds to a Ruby Hash
|
146
|
-
def to_ruby
|
139
|
+
def to_ruby
|
147
140
|
rep = {}
|
148
141
|
members.each do |pair|
|
149
142
|
rep[pair.name.to_ruby] = pair.value.to_ruby
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Grammar for JSON data representation
|
2
4
|
require 'rley' # Load the gem
|
3
5
|
|
@@ -16,11 +18,11 @@ builder = Rley::Syntax::GrammarBuilder.new do
|
|
16
18
|
rule 'JSON_text' => 'value'
|
17
19
|
rule 'value' => 'false'
|
18
20
|
rule 'value' => 'null'
|
19
|
-
rule 'value' => 'true'
|
21
|
+
rule 'value' => 'true'
|
20
22
|
rule 'value' => 'object'
|
21
23
|
rule 'value' => 'array'
|
22
24
|
rule 'value' => 'number'
|
23
|
-
rule 'value' => 'string'
|
25
|
+
rule 'value' => 'string'
|
24
26
|
rule 'object' => %w[begin-object member-list end-object]
|
25
27
|
rule 'object' => %w[begin-object end-object]
|
26
28
|
# Next rule is an example of a left recursive rule
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# File: json_lexer.rb
|
2
4
|
# Lexer for the JSON data format
|
3
5
|
require 'rley' # Load the gem
|
@@ -26,19 +28,19 @@ class JSONLexer
|
|
26
28
|
@line_start = 0
|
27
29
|
end
|
28
30
|
|
29
|
-
def tokens
|
31
|
+
def tokens
|
30
32
|
tok_sequence = []
|
31
33
|
until @scanner.eos?
|
32
34
|
token = _next_token
|
33
35
|
tok_sequence << token unless token.nil?
|
34
36
|
end
|
35
37
|
|
36
|
-
|
38
|
+
tok_sequence
|
37
39
|
end
|
38
40
|
|
39
41
|
private
|
40
42
|
|
41
|
-
def _next_token
|
43
|
+
def _next_token
|
42
44
|
token = nil
|
43
45
|
skip_whitespaces
|
44
46
|
curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
|
@@ -56,7 +58,7 @@ class JSONLexer
|
|
56
58
|
keyw = scanner.scan(/false|true|null/)
|
57
59
|
if keyw.nil?
|
58
60
|
invalid_keyw = scanner.scan(/\w+/)
|
59
|
-
raise ScanError
|
61
|
+
raise ScanError, "Invalid keyword: #{invalid_keyw}"
|
60
62
|
else
|
61
63
|
token = build_token(keyw, keyw)
|
62
64
|
end
|
@@ -66,7 +68,7 @@ class JSONLexer
|
|
66
68
|
value = scanner.scan(/([^"\\]|\\.)*/)
|
67
69
|
end_delimiter = scanner.getch
|
68
70
|
err_msg = 'No closing quotes (") found'
|
69
|
-
raise ScanError
|
71
|
+
raise ScanError, err_msg if end_delimiter.nil?
|
70
72
|
|
71
73
|
token = build_token(value, 'string')
|
72
74
|
|
@@ -79,12 +81,12 @@ class JSONLexer
|
|
79
81
|
erroneous = curr_ch.nil? ? '' : curr_ch
|
80
82
|
sequel = scanner.scan(/.{1,20}/)
|
81
83
|
erroneous += sequel unless sequel.nil?
|
82
|
-
raise ScanError
|
84
|
+
raise ScanError, "Unknown token #{erroneous}"
|
83
85
|
end # case
|
84
86
|
break unless token.nil? && (curr_ch = scanner.getch)
|
85
87
|
end
|
86
88
|
|
87
|
-
|
89
|
+
token
|
88
90
|
end
|
89
91
|
|
90
92
|
def build_token(lexeme, token)
|
@@ -92,7 +94,7 @@ class JSONLexer
|
|
92
94
|
Rley::Lexical::Token.new(lexeme, token, pos)
|
93
95
|
end
|
94
96
|
|
95
|
-
def skip_whitespaces
|
97
|
+
def skip_whitespaces
|
96
98
|
matched = scanner.scan(/[ \t\f\n\r]+/)
|
97
99
|
return if matched.nil?
|
98
100
|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# File: json_minifier.rb
|
2
4
|
|
3
5
|
|
@@ -33,7 +35,7 @@ class JSONMinifier
|
|
33
35
|
lexeme = aTerm.token.lexeme
|
34
36
|
literal = if aTerm.symbol.name == 'string'
|
35
37
|
# String values are delimited by double quotes
|
36
|
-
|
38
|
+
"\"#{lexeme}\""
|
37
39
|
else
|
38
40
|
lexeme
|
39
41
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require_relative 'calc_ast_nodes'
|
2
4
|
|
3
5
|
# The purpose of a CalcASTBuilder is to build piece by piece an AST
|
@@ -13,8 +15,8 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
13
15
|
}.freeze
|
14
16
|
|
15
17
|
protected
|
16
|
-
|
17
|
-
def terminal2node
|
18
|
+
|
19
|
+
def terminal2node
|
18
20
|
Terminal2NodeClass
|
19
21
|
end
|
20
22
|
|
@@ -24,7 +26,9 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
24
26
|
operator_node.children << theChildren[2]
|
25
27
|
return operator_node
|
26
28
|
end
|
27
|
-
|
29
|
+
|
30
|
+
# rubocop: disable Naming/VariableNumber
|
31
|
+
|
28
32
|
# rule 'expression' => 'simple_expression'
|
29
33
|
def reduce_expression_0(_production, aRange, theTokens, theChildren)
|
30
34
|
return_first_child(aRange, theTokens, theChildren)
|
@@ -33,32 +37,32 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
33
37
|
# rule 'simple_expression' => 'term'
|
34
38
|
def reduce_simple_expression_0(_production, aRange, theTokens, theChildren)
|
35
39
|
return_first_child(aRange, theTokens, theChildren)
|
36
|
-
end
|
40
|
+
end
|
37
41
|
|
38
42
|
# rule 'simple_expression' => %w[simple_expression add_operator term]
|
39
43
|
def reduce_simple_expression_1(_production, _range, _tokens, theChildren)
|
40
44
|
reduce_binary_operator(theChildren)
|
41
45
|
end
|
42
|
-
|
43
|
-
# rule 'term' => 'factor'
|
46
|
+
|
47
|
+
# rule 'term' => 'factor'
|
44
48
|
def reduce_term_0(_production, aRange, theTokens, theChildren)
|
45
49
|
return_first_child(aRange, theTokens, theChildren)
|
46
|
-
end
|
50
|
+
end
|
47
51
|
|
48
52
|
# rule 'term' => %w[term mul_operator factor]
|
49
53
|
def reduce_term_1(_production, _range, _tokens, theChildren)
|
50
54
|
reduce_binary_operator(theChildren)
|
51
55
|
end
|
52
|
-
|
56
|
+
|
53
57
|
# rule 'factor' => 'NUMBER'
|
54
58
|
def reduce_factor_0(_aProd, aRange, theTokens, theChildren)
|
55
59
|
return_first_child(aRange, theTokens, theChildren)
|
56
60
|
end
|
57
|
-
|
61
|
+
|
58
62
|
# # rule 'factor' => %w[LPAREN expression RPAREN]
|
59
63
|
def reduce_factor_1(_aProd, aRange, theTokens, theChildren)
|
60
64
|
return_second_child(aRange, theTokens, theChildren)
|
61
|
-
end
|
65
|
+
end
|
62
66
|
|
63
67
|
# rule 'add_operator' => 'PLUS'
|
64
68
|
def reduce_add_operator_0(_production, _range, _tokens, theChildren)
|
@@ -79,5 +83,6 @@ class CalcASTBuilder < Rley::ParseRep::ASTBaseBuilder
|
|
79
83
|
def reduce_mul_operator_1(_production, _range, _tokens, theChildren)
|
80
84
|
return CalcDivideNode.new(theChildren[0].symbol)
|
81
85
|
end
|
86
|
+
# rubocop: enable Naming/VariableNumber
|
82
87
|
end # class
|
83
88
|
# End of file
|
@@ -1,7 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# Classes that implement nodes of Abstract Syntax Trees (AST) representing
|
2
4
|
# calculator parse results.
|
3
5
|
|
4
|
-
|
5
6
|
CalcTerminalNode = Struct.new(:token, :value, :position) do
|
6
7
|
def initialize(aToken, aPosition)
|
7
8
|
self.token = aToken
|
@@ -14,23 +15,23 @@ CalcTerminalNode = Struct.new(:token, :value, :position) do
|
|
14
15
|
self.value = aLiteral.dup
|
15
16
|
end
|
16
17
|
|
17
|
-
def symbol
|
18
|
+
def symbol
|
18
19
|
token.terminal
|
19
20
|
end
|
20
21
|
|
21
|
-
def interpret
|
22
|
-
|
22
|
+
def interpret
|
23
|
+
value
|
23
24
|
end
|
24
|
-
|
25
|
-
def done!
|
25
|
+
|
26
|
+
def done!
|
26
27
|
# Do nothing
|
27
28
|
end
|
28
|
-
|
29
|
+
|
29
30
|
# Part of the 'visitee' role in Visitor design pattern.
|
30
31
|
# @param aVisitor[ParseTreeVisitor] the visitor
|
31
32
|
def accept(aVisitor)
|
32
33
|
aVisitor.visit_terminal(self)
|
33
|
-
end
|
34
|
+
end
|
34
35
|
end
|
35
36
|
|
36
37
|
class CalcNumberNode < CalcTerminalNode
|
@@ -51,30 +52,26 @@ class CalcCompositeNode
|
|
51
52
|
|
52
53
|
def initialize(aSymbol)
|
53
54
|
@symbol = aSymbol
|
54
|
-
@children = []
|
55
|
+
@children = []
|
55
56
|
end
|
56
|
-
|
57
|
-
def done!
|
57
|
+
|
58
|
+
def done!
|
58
59
|
# Do nothing
|
59
|
-
end
|
60
|
+
end
|
60
61
|
|
61
62
|
# Part of the 'visitee' role in Visitor design pattern.
|
62
63
|
# @param aVisitor[ParseTreeVisitor] the visitor
|
63
64
|
def accept(aVisitor)
|
64
65
|
aVisitor.visit_nonterminal(self)
|
65
66
|
end
|
66
|
-
|
67
|
+
|
67
68
|
alias subnodes children
|
68
69
|
end # class
|
69
70
|
|
70
71
|
class CalcUnaryOpNode < CalcCompositeNode
|
71
|
-
def initialize(aSymbol)
|
72
|
-
super(aSymbol)
|
73
|
-
end
|
74
|
-
|
75
72
|
# Convert this tree node in a simpler Ruby representation.
|
76
73
|
# Basically a Calc object corresponds to a Ruhy Hash
|
77
|
-
def to_ruby
|
74
|
+
def to_ruby
|
78
75
|
rep = {}
|
79
76
|
members.each do |pair|
|
80
77
|
rep[pair.name.to_ruby] = pair.value.to_ruby
|
@@ -90,13 +87,9 @@ class CalcNegateNode < CalcUnaryOpNode
|
|
90
87
|
end # class
|
91
88
|
|
92
89
|
class CalcBinaryOpNode < CalcCompositeNode
|
93
|
-
def initialize(aSymbol)
|
94
|
-
super(aSymbol)
|
95
|
-
end
|
96
|
-
|
97
90
|
protected
|
98
|
-
|
99
|
-
def retrieve_operands
|
91
|
+
|
92
|
+
def retrieve_operands
|
100
93
|
operands = []
|
101
94
|
children.each do |child|
|
102
95
|
oper = child.respond_to?(:interpret) ? child.interpret : child
|
@@ -109,42 +102,37 @@ end # class
|
|
109
102
|
|
110
103
|
class CalcAddNode < CalcBinaryOpNode
|
111
104
|
# TODO
|
112
|
-
def interpret
|
105
|
+
def interpret
|
113
106
|
operands = retrieve_operands
|
114
107
|
|
115
|
-
|
116
|
-
return sum
|
108
|
+
operands[0] + operands[1]
|
117
109
|
end
|
118
110
|
end # class
|
119
111
|
|
120
|
-
|
121
112
|
class CalcSubtractNode < CalcBinaryOpNode
|
122
113
|
# TODO
|
123
|
-
def interpret
|
114
|
+
def interpret
|
124
115
|
operands = retrieve_operands
|
125
116
|
|
126
|
-
|
127
|
-
return substraction
|
117
|
+
operands[0] - operands[1]
|
128
118
|
end
|
129
119
|
end # class
|
130
120
|
|
131
121
|
class CalcMultiplyNode < CalcBinaryOpNode
|
132
122
|
# TODO
|
133
|
-
def interpret
|
123
|
+
def interpret
|
134
124
|
operands = retrieve_operands
|
135
|
-
|
136
|
-
return multiplication
|
125
|
+
operands[0] * operands[1]
|
137
126
|
end
|
138
127
|
end # class
|
139
128
|
|
140
129
|
class CalcDivideNode < CalcBinaryOpNode
|
141
130
|
# TODO
|
142
|
-
def interpret
|
131
|
+
def interpret
|
143
132
|
operands = retrieve_operands
|
144
133
|
numerator = operands[0].to_f
|
145
134
|
denominator = operands[1]
|
146
|
-
|
147
|
-
return division
|
135
|
+
numerator / denominator
|
148
136
|
end
|
149
137
|
end # class
|
150
138
|
# End of file
|