rley 0.8.00 → 0.8.05

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +47 -3
  3. data/CHANGELOG.md +32 -4
  4. data/examples/NLP/pico_en_demo.rb +2 -2
  5. data/examples/data_formats/JSON/README.md +34 -0
  6. data/examples/data_formats/JSON/sample01.json +3 -0
  7. data/examples/data_formats/JSON/sample01.svg +36 -0
  8. data/examples/data_formats/JSON/sample02.json +6 -0
  9. data/examples/data_formats/JSON/sample02.svg +128 -0
  10. data/examples/data_formats/JSON/sample03.json +88 -0
  11. data/examples/general/calc_iter1/README.md +26 -0
  12. data/examples/general/calc_iter2/README.md +55 -0
  13. data/examples/general/general_examples.md +37 -0
  14. data/examples/tokenizer/README.md +46 -0
  15. data/examples/tokenizer/loxxy_raw_scanner.rex +98 -0
  16. data/examples/tokenizer/loxxy_raw_scanner.rex.rb +256 -0
  17. data/examples/tokenizer/loxxy_tokenizer.rb +94 -0
  18. data/examples/tokenizer/run_tokenizer.rb +29 -0
  19. data/lib/rley/constants.rb +1 -1
  20. data/lib/rley/lexical/literal.rb +29 -0
  21. data/lib/rley/lexical/token.rb +7 -4
  22. data/lib/rley/notation/all_notation_nodes.rb +3 -1
  23. data/lib/rley/notation/ast_builder.rb +185 -191
  24. data/lib/rley/notation/ast_node.rb +5 -5
  25. data/lib/rley/notation/ast_visitor.rb +3 -1
  26. data/lib/rley/notation/grammar.rb +1 -1
  27. data/lib/rley/notation/grammar_builder.rb +87 -33
  28. data/lib/rley/notation/grouping_node.rb +1 -1
  29. data/lib/rley/notation/parser.rb +56 -56
  30. data/lib/rley/notation/sequence_node.rb +3 -3
  31. data/lib/rley/notation/symbol_node.rb +2 -2
  32. data/lib/rley/notation/tokenizer.rb +3 -15
  33. data/lib/rley/parse_rep/ast_base_builder.rb +35 -4
  34. data/lib/rley/parser/gfg_chart.rb +5 -4
  35. data/lib/rley/parser/gfg_earley_parser.rb +1 -1
  36. data/lib/rley/syntax/base_grammar_builder.rb +8 -2
  37. data/lib/rley/syntax/match_closest.rb +7 -7
  38. data/lib/rley.rb +1 -1
  39. data/spec/rley/lexical/literal_spec.rb +33 -0
  40. data/spec/rley/lexical/token_spec.rb +15 -4
  41. data/spec/rley/notation/grammar_builder_spec.rb +57 -50
  42. data/spec/rley/notation/parser_spec.rb +183 -184
  43. data/spec/rley/notation/tokenizer_spec.rb +98 -104
  44. data/spec/rley/parser/dangling_else_spec.rb +20 -20
  45. data/spec/rley/parser/gfg_chart_spec.rb +0 -1
  46. data/spec/rley/parser/gfg_earley_parser_spec.rb +166 -147
  47. data/spec/rley/parser/gfg_parsing_spec.rb +2 -2
  48. data/spec/rley/syntax/base_grammar_builder_spec.rb +7 -8
  49. data/spec/rley/syntax/grammar_spec.rb +6 -9
  50. data/spec/rley/syntax/match_closest_spec.rb +4 -4
  51. metadata +19 -9
  52. data/lib/rley/parser/parse_tracer.rb +0 -103
  53. data/lib/rley/syntax/literal.rb +0 -20
  54. data/lib/rley/syntax/verbatim_symbol.rb +0 -27
  55. data/spec/rley/syntax/literal_spec.rb +0 -31
  56. data/spec/rley/syntax/verbatim_symbol_spec.rb +0 -38
@@ -1,191 +1,185 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../parse_rep/ast_base_builder'
4
- require_relative '../engine'
5
- require_relative 'all_notation_nodes'
6
-
7
- module Rley
8
- module Notation
9
- # The purpose of ASTBuilder is to build piece by piece an AST
10
- # (Abstract Syntax Tree) from a sequence of input tokens and
11
- # visit events produced by walking over a GFGParsing object.
12
- class ASTBuilder < Rley::ParseRep::ASTBaseBuilder
13
- unless defined?(Name2special)
14
- # Mapping Token name => operator | separator | delimiter characters
15
- # @return [Hash{String => String}]
16
- Name2special = {
17
- 'COMMA' => ',',
18
- 'ELLIPSIS' => '..',
19
- 'LEFT_BRACE' => '{',
20
- 'LEFT_PAREN' => '(',
21
- 'PLUS' => '+',
22
- 'QUESTION_MARK' => '?',
23
- 'RIGHT_BRACE' => '}',
24
- 'RIGHT_PAREN' => ')',
25
- 'STAR' => '*'
26
- }.freeze
27
- end
28
-
29
- # Create a new AST builder instance.
30
- # @param theTokens [Array<Rley::Lexical::Token>] The sequence of input tokens.
31
- def initialize(theTokens)
32
- super(theTokens)
33
- end
34
-
35
- protected
36
-
37
- def terminal2node
38
- Terminal2NodeClass
39
- end
40
-
41
- # Method override
42
- def new_leaf_node(_production, _terminal, aTokenPosition, aToken)
43
- Rley::PTree::TerminalNode.new(aToken, aTokenPosition)
44
- end
45
-
46
- # Factory method for creating a parent node object.
47
- # @param aProduction [Production] Production rule
48
- # @param aRange [Range] Range of tokens matched by the rule
49
- # @param theTokens [Array] The input tokens
50
- # @param theChildren [Array] Children nodes (one per rhs symbol)
51
- def new_parent_node(aProduction, aRange, theTokens, theChildren)
52
- mth_name = method_name(aProduction.name)
53
- if respond_to?(mth_name, true)
54
- node = send(mth_name, aProduction, aRange, theTokens, theChildren)
55
- else
56
- # Default action...
57
- node = case aProduction.rhs.size
58
- when 0
59
- return_epsilon(aRange, theTokens, theChildren)
60
- when 1
61
- return_first_child(aRange, theTokens, theChildren)
62
- else
63
- node = Rley::PTree::NonTerminalNode.new(aProduction.lhs, aRange)
64
- theChildren&.reverse_each do |child|
65
- node.add_subnode(child) if child
66
- end
67
-
68
- node
69
- end
70
- end
71
-
72
- node
73
- end
74
-
75
- # Return the AST node corresponding to the second symbol in the rhs
76
- def reduce_to_2nd_symbol(_production, _range, _tokens, theChildren)
77
- theChildren[1]
78
- end
79
-
80
- #####################################
81
- # SEMANTIC ACTIONS
82
- #####################################
83
-
84
- # rule('rhs' => 'member_seq').tag 'sequence'
85
- def reduce_sequence(_production, _range, _tokens, theChildren)
86
- if theChildren[0].size == 1
87
- theChildren[0].first
88
- else
89
- SequenceNode.new(theChildren[0].first.position, theChildren[0], nil)
90
- end
91
- end
92
-
93
- # rule('member_seq' => 'member_seq member').tag 'more_members'
94
- def reduce_more_members(_production, _range, _tokens, theChildren)
95
- theChildren[0] << theChildren[1]
96
- end
97
-
98
- # rule('member_seq' => 'member')
99
- def reduce_one_member(_production, _range, _tokens, theChildren)
100
- [theChildren[0]]
101
- end
102
-
103
- # rule('strait_member' => 'base_member annotation')
104
- def reduce_annotated_member(_production, _range, _tokens, theChildren)
105
- theChildren[0].annotation = theChildren[1]
106
-
107
- theChildren[0]
108
- end
109
-
110
- # rule('base_member' => 'SYMBOL')
111
- def reduce_symbol(_production, _range, _tokens, theChildren)
112
- SymbolNode.new(theChildren[0].token.position, theChildren[0].token.lexeme)
113
- end
114
-
115
- # rule('base_member' => 'LEFT_PAREN member_seq RIGHT_PAREN')
116
- def reduce_grouping(_production, _range, tokens, theChildren)
117
- if theChildren[1].size == 1
118
- theChildren[1].first
119
- else
120
- rank = theChildren[0].range.high
121
- pos = tokens[rank].position
122
- GroupingNode.new(pos, theChildren[1], nil)
123
- end
124
- end
125
-
126
- # rule('quantified_member' => 'base_member quantifier')
127
- def reduce_quantified_member(_production, _range, _tokens, theChildren)
128
- theChildren[0].repetition = theChildren[1]
129
- theChildren[0]
130
- end
131
-
132
- # rule('quantifier' => 'QUESTION_MARK')
133
- def reduce_question_mark(_production, _range, _tokens, _theChildren)
134
- :zero_or_one
135
- end
136
-
137
- # rule('quantifier' => 'STAR')
138
- def reduce_star(_production, _range, _tokens, _theChildren)
139
- :zero_or_more
140
- end
141
-
142
- # rule('quantifier' => 'PLUS')
143
- def reduce_plus(_production, _range, _tokens, _theChildren)
144
- :one_or_more
145
- end
146
-
147
- # rule('annotation' => 'LEFT_BRACE mapping RIGHT_BRACE').tag ''
148
- def reduce_annotation(_production, _range, _tokens, theChildren)
149
- theChildren[1]
150
- end
151
-
152
- # rule('mapping' => 'mapping COMMA key_value')
153
- def reduce_more_pairs(_production, _range, _tokens, theChildren)
154
- hsh = theChildren[0]
155
- hsh[theChildren[2].first] = theChildren[2].last
156
-
157
- hsh
158
- end
159
-
160
- # rule('mapping' => 'key_value').tag 'one_pair'
161
- def reduce_one_pair(_production, _range, _tokens, theChildren)
162
- { theChildren[0].first => theChildren[0].last }
163
- end
164
-
165
- # rule('key_value' => 'KEY value')
166
- def reduce_raw_pair(_production, _range, _tokens, theChildren)
167
- key = theChildren[0].token.lexeme
168
- value = if theChildren[1].kind_of?(Rley::PTree::TerminalNode)
169
- theChildren[1].token.lexeme
170
- else
171
- theChildren[1]
172
- end
173
- [key, value]
174
- end
175
-
176
- # rule('range' => 'INT_LIT ELLIPSIS INT_LIT')
177
- def reduce_bound_range(_production, _range, _tokens, theChildren)
178
- low = theChildren[0].token.lexeme
179
- high = theChildren[2].token.lexeme
180
- case [low, high]
181
- when ['0', '1']
182
- :zero_or_one
183
- when ['1', '1']
184
- :exactly_one
185
- else
186
- Range.new(low.to_i, high.to_i)
187
- end
188
- end
189
- end # class
190
- end # module
191
- end # module
1
+ # frozen_string_literal: true
2
+
3
+ require_relative '../parse_rep/ast_base_builder'
4
+ require_relative '../engine'
5
+ require_relative 'all_notation_nodes'
6
+
7
+ module Rley
8
+ module Notation
9
+ # The purpose of ASTBuilder is to build piece by piece an AST
10
+ # (Abstract Syntax Tree) from a sequence of input tokens and
11
+ # visit events produced by walking over a GFGParsing object.
12
+ class ASTBuilder < Rley::ParseRep::ASTBaseBuilder
13
+ unless defined?(Name2special)
14
+ # Mapping Token name => operator | separator | delimiter characters
15
+ # @return [Hash{String => String}]
16
+ Name2special = {
17
+ 'COMMA' => ',',
18
+ 'ELLIPSIS' => '..',
19
+ 'LEFT_BRACE' => '{',
20
+ 'LEFT_PAREN' => '(',
21
+ 'PLUS' => '+',
22
+ 'QUESTION_MARK' => '?',
23
+ 'RIGHT_BRACE' => '}',
24
+ 'RIGHT_PAREN' => ')',
25
+ 'STAR' => '*'
26
+ }.freeze
27
+ end
28
+
29
+ protected
30
+
31
+ def terminal2node
32
+ Terminal2NodeClass
33
+ end
34
+
35
+ # Method override
36
+ def new_leaf_node(_production, _terminal, aTokenPosition, aToken)
37
+ Rley::PTree::TerminalNode.new(aToken, aTokenPosition)
38
+ end
39
+
40
+ # Factory method for creating a parent node object.
41
+ # @param aProduction [Production] Production rule
42
+ # @param aRange [Range] Range of tokens matched by the rule
43
+ # @param theTokens [Array] The input tokens
44
+ # @param theChildren [Array] Children nodes (one per rhs symbol)
45
+ def new_parent_node(aProduction, aRange, theTokens, theChildren)
46
+ mth_name = method_name(aProduction.name)
47
+ if respond_to?(mth_name, true)
48
+ node = send(mth_name, aProduction, aRange, theTokens, theChildren)
49
+ else
50
+ # Default action...
51
+ node = case aProduction.rhs.size
52
+ when 0
53
+ return_epsilon(aRange, theTokens, theChildren)
54
+ when 1
55
+ return_first_child(aRange, theTokens, theChildren)
56
+ else
57
+ node = Rley::PTree::NonTerminalNode.new(aProduction.lhs, aRange)
58
+ theChildren&.reverse_each do |child|
59
+ node.add_subnode(child) if child
60
+ end
61
+
62
+ node
63
+ end
64
+ end
65
+
66
+ node
67
+ end
68
+
69
+ # Return the AST node corresponding to the second symbol in the rhs
70
+ def reduce_to_2nd_symbol(_production, _range, _tokens, theChildren)
71
+ theChildren[1]
72
+ end
73
+
74
+ #####################################
75
+ # SEMANTIC ACTIONS
76
+ #####################################
77
+
78
+ # rule('rhs' => 'member_seq').tag 'sequence'
79
+ def reduce_sequence(_production, _range, _tokens, theChildren)
80
+ if theChildren[0].size == 1
81
+ theChildren[0].first
82
+ else
83
+ SequenceNode.new(theChildren[0].first.position, theChildren[0], nil)
84
+ end
85
+ end
86
+
87
+ # rule('member_seq' => 'member_seq member').tag 'more_members'
88
+ def reduce_more_members(_production, _range, _tokens, theChildren)
89
+ theChildren[0] << theChildren[1]
90
+ end
91
+
92
+ # rule('member_seq' => 'member')
93
+ def reduce_one_member(_production, _range, _tokens, theChildren)
94
+ [theChildren[0]]
95
+ end
96
+
97
+ # rule('strait_member' => 'base_member annotation')
98
+ def reduce_annotated_member(_production, _range, _tokens, theChildren)
99
+ theChildren[0].annotation = theChildren[1]
100
+
101
+ theChildren[0]
102
+ end
103
+
104
+ # rule('base_member' => 'SYMBOL')
105
+ def reduce_symbol(_production, _range, _tokens, theChildren)
106
+ SymbolNode.new(theChildren[0].token.position, theChildren[0].token.lexeme)
107
+ end
108
+
109
+ # rule('base_member' => 'LEFT_PAREN member_seq RIGHT_PAREN')
110
+ def reduce_grouping(_production, _range, tokens, theChildren)
111
+ if theChildren[1].size == 1
112
+ theChildren[1].first
113
+ else
114
+ rank = theChildren[0].range.high
115
+ pos = tokens[rank].position
116
+ GroupingNode.new(pos, theChildren[1], nil)
117
+ end
118
+ end
119
+
120
+ # rule('quantified_member' => 'base_member quantifier')
121
+ def reduce_quantified_member(_production, _range, _tokens, theChildren)
122
+ theChildren[0].repetition = theChildren[1]
123
+ theChildren[0]
124
+ end
125
+
126
+ # rule('quantifier' => 'QUESTION_MARK')
127
+ def reduce_question_mark(_production, _range, _tokens, _theChildren)
128
+ :zero_or_one
129
+ end
130
+
131
+ # rule('quantifier' => 'STAR')
132
+ def reduce_star(_production, _range, _tokens, _theChildren)
133
+ :zero_or_more
134
+ end
135
+
136
+ # rule('quantifier' => 'PLUS')
137
+ def reduce_plus(_production, _range, _tokens, _theChildren)
138
+ :one_or_more
139
+ end
140
+
141
+ # rule('annotation' => 'LEFT_BRACE mapping RIGHT_BRACE').tag ''
142
+ def reduce_annotation(_production, _range, _tokens, theChildren)
143
+ theChildren[1]
144
+ end
145
+
146
+ # rule('mapping' => 'mapping COMMA key_value')
147
+ def reduce_more_pairs(_production, _range, _tokens, theChildren)
148
+ hsh = theChildren[0]
149
+ hsh[theChildren[2].first] = theChildren[2].last
150
+
151
+ hsh
152
+ end
153
+
154
+ # rule('mapping' => 'key_value').tag 'one_pair'
155
+ def reduce_one_pair(_production, _range, _tokens, theChildren)
156
+ { theChildren[0].first => theChildren[0].last }
157
+ end
158
+
159
+ # rule('key_value' => 'KEY value')
160
+ def reduce_raw_pair(_production, _range, _tokens, theChildren)
161
+ key = theChildren[0].token.lexeme
162
+ value = if theChildren[1].kind_of?(Rley::PTree::TerminalNode)
163
+ theChildren[1].token.lexeme
164
+ else
165
+ theChildren[1]
166
+ end
167
+ [key, value]
168
+ end
169
+
170
+ # rule('range' => 'INT_LIT ELLIPSIS INT_LIT')
171
+ def reduce_bound_range(_production, _range, _tokens, theChildren)
172
+ low = theChildren[0].token.lexeme
173
+ high = theChildren[2].token.lexeme
174
+ case [low, high]
175
+ when %w[0 1]
176
+ :zero_or_one
177
+ when %w[1 1]
178
+ :exactly_one
179
+ else
180
+ Range.new(low.to_i, high.to_i)
181
+ end
182
+ end
183
+ end # class
184
+ end # module
185
+ end # module
@@ -8,25 +8,25 @@ module Rley
8
8
  class ASTNode
9
9
  # @return [Rley::Lexical::Position] Position of the entry in the input stream.
10
10
  attr_reader :position
11
-
11
+
12
12
  # @return [Symbol]
13
13
  attr_accessor :repetition
14
14
 
15
15
  # @return [Hash]
16
- attr_reader :annotation
16
+ attr_reader :annotation
17
17
 
18
18
  # @param aPosition [Rley::Lexical::Position] Position of the entry in the input stream.
19
19
  def initialize(aPosition)
20
20
  @position = aPosition
21
21
  @repetition = :exactly_one
22
- @annotation = {}
22
+ @annotation = {}
23
23
  end
24
-
24
+
25
25
  def annotation=(aMapping)
26
26
  repeat_key = 'repeat'
27
27
  @repetition = aMapping.delete(repeat_key) if aMapping.include?(repeat_key)
28
28
  @annotation = aMapping
29
- end
29
+ end
30
30
 
31
31
  # Notification that the parsing has successfully completed
32
32
  def done!
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rley
2
4
  module Notation
3
5
  class ASTVisitor
@@ -110,4 +112,4 @@ module Rley
110
112
  end
111
113
  end # class
112
114
  end # module
113
- end # module
115
+ end # module
@@ -46,4 +46,4 @@ module Rley
46
46
  # And now build the Rley Grammar Notation (RGN) grammar...
47
47
  RGNGrammar = builder.grammar
48
48
  end # module
49
- end # module
49
+ end # module