rley 0.4.06 → 0.4.07

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +22 -13
  3. data/.travis.yml +8 -7
  4. data/CHANGELOG.md +9 -0
  5. data/README.md +9 -10
  6. data/examples/NLP/mini_en_demo.rb +7 -7
  7. data/examples/data_formats/JSON/cli_options.rb +5 -6
  8. data/examples/data_formats/JSON/{JSON_demo.rb → json_demo.rb} +2 -2
  9. data/examples/data_formats/JSON/{JSON_grammar.rb → json_grammar.rb} +10 -10
  10. data/examples/data_formats/JSON/{JSON_lexer.rb → json_lexer.rb} +17 -22
  11. data/examples/data_formats/JSON/{JSON_parser.rb → json_parser.rb} +2 -2
  12. data/examples/general/calc/calc_demo.rb +1 -1
  13. data/examples/general/calc/calc_grammar.rb +5 -5
  14. data/examples/general/calc/calc_lexer.rb +14 -17
  15. data/examples/general/calc/calc_parser.rb +2 -2
  16. data/lib/rley/constants.rb +1 -1
  17. data/lib/rley/formatter/asciitree.rb +15 -16
  18. data/lib/rley/formatter/bracket_notation.rb +3 -6
  19. data/lib/rley/formatter/debug.rb +0 -1
  20. data/lib/rley/formatter/json.rb +0 -1
  21. data/lib/rley/gfg/grm_flow_graph.rb +11 -10
  22. data/lib/rley/parse_forest_visitor.rb +0 -3
  23. data/lib/rley/parse_tree_visitor.rb +0 -3
  24. data/lib/rley/parser/error_reason.rb +1 -5
  25. data/lib/rley/parser/gfg_chart.rb +2 -2
  26. data/lib/rley/parser/gfg_earley_parser.rb +1 -2
  27. data/lib/rley/parser/gfg_parsing.rb +3 -7
  28. data/lib/rley/parser/parse_entry.rb +0 -1
  29. data/lib/rley/parser/parse_entry_set.rb +15 -16
  30. data/lib/rley/parser/parse_forest_builder.rb +8 -23
  31. data/lib/rley/parser/parse_state.rb +1 -1
  32. data/lib/rley/parser/parse_tree_builder.rb +2 -30
  33. data/lib/rley/parser/parse_tree_factory.rb +1 -1
  34. data/lib/rley/parser/parse_walker_factory.rb +3 -6
  35. data/lib/rley/parser/state_set.rb +0 -1
  36. data/lib/rley/ptree/parse_tree.rb +0 -1
  37. data/lib/rley/ptree/terminal_node.rb +4 -1
  38. data/lib/rley/rley_error.rb +1 -1
  39. data/lib/rley/sppf/composite_node.rb +0 -1
  40. data/lib/rley/sppf/parse_forest.rb +0 -1
  41. data/lib/rley/syntax/grammar.rb +5 -9
  42. data/lib/rley/syntax/grammar_builder.rb +8 -11
  43. data/lib/rley/syntax/grm_symbol.rb +0 -1
  44. data/lib/rley/syntax/production.rb +5 -4
  45. data/lib/rley/tokens/token_range.rb +0 -1
  46. data/spec/rley/formatter/bracket_notation_spec.rb +3 -1
  47. data/spec/rley/gfg/grm_flow_graph_spec.rb +15 -46
  48. data/spec/rley/gfg/item_vertex_spec.rb +1 -1
  49. data/spec/rley/parse_forest_visitor_spec.rb +1 -1
  50. data/spec/rley/parse_tree_visitor_spec.rb +2 -2
  51. data/spec/rley/parser/error_reason_spec.rb +19 -14
  52. data/spec/rley/parser/gfg_chart_spec.rb +1 -1
  53. data/spec/rley/parser/gfg_earley_parser_spec.rb +15 -15
  54. data/spec/rley/parser/gfg_parsing_spec.rb +3 -3
  55. data/spec/rley/parser/groucho_spec.rb +6 -7
  56. data/spec/rley/parser/parse_forest_builder_spec.rb +5 -5
  57. data/spec/rley/parser/parse_forest_factory_spec.rb +5 -5
  58. data/spec/rley/parser/parse_state_spec.rb +8 -0
  59. data/spec/rley/parser/parse_tracer_spec.rb +1 -1
  60. data/spec/rley/parser/parse_tree_builder_spec.rb +26 -29
  61. data/spec/rley/parser/parse_tree_factory_spec.rb +5 -5
  62. data/spec/rley/parser/parse_walker_factory_spec.rb +5 -5
  63. data/spec/rley/ptree/parse_tree_node_spec.rb +2 -2
  64. data/spec/rley/ptree/terminal_node_spec.rb +1 -1
  65. data/spec/rley/support/ambiguous_grammar_helper.rb +1 -1
  66. data/spec/rley/support/expectation_helper.rb +0 -1
  67. data/spec/rley/support/grammar_abc_helper.rb +1 -1
  68. data/spec/rley/support/grammar_ambig01_helper.rb +2 -3
  69. data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
  70. data/spec/rley/support/grammar_l0_helper.rb +7 -8
  71. data/spec/rley/support/grammar_pb_helper.rb +3 -4
  72. data/spec/rley/support/grammar_sppf_helper.rb +4 -4
  73. data/spec/rley/syntax/grammar_builder_spec.rb +5 -4
  74. data/spec/rley/syntax/grammar_spec.rb +10 -11
  75. data/spec/rley/syntax/symbol_seq_spec.rb +2 -2
  76. data/spec/rley/syntax/terminal_spec.rb +1 -1
  77. metadata +31 -31
@@ -1,7 +1,7 @@
1
1
  # File: calc_lexer.rb
2
2
  # Lexer for a basic arithmetical expression parser
3
3
  require 'strscan'
4
- require 'rley' # Load the gem
4
+ require 'rley' # Load the gem
5
5
 
6
6
 
7
7
  class CalcLexer
@@ -16,21 +16,20 @@ class CalcLexer
16
16
  '+' => 'PLUS',
17
17
  '-' => 'MINUS',
18
18
  '*' => 'STAR',
19
- '/' => 'DIVIDE',
20
- }
19
+ '/' => 'DIVIDE'
20
+ }.freeze
21
21
 
22
- class ScanError < StandardError ; end
22
+ class ScanError < StandardError; end
23
23
 
24
- public
25
24
  def initialize(source, aGrammar)
26
25
  @scanner = StringScanner.new(source)
27
26
  @name2symbol = aGrammar.name2symbol
28
- @lineno = 1
27
+ @lineno = 1
29
28
  end
30
29
 
31
30
  def tokens()
32
31
  tok_sequence = []
33
- until @scanner.eos? do
32
+ until @scanner.eos?
34
33
  token = _next_token
35
34
  tok_sequence << token unless token.nil?
36
35
  end
@@ -38,13 +37,14 @@ public
38
37
  return tok_sequence
39
38
  end
40
39
 
41
- private
40
+ private
41
+
42
42
  def _next_token()
43
43
  token = nil
44
44
  skip_whitespaces
45
45
  curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
46
46
 
47
- begin
47
+ loop do
48
48
  break if curr_ch.nil?
49
49
 
50
50
  case curr_ch
@@ -66,15 +66,14 @@ private
66
66
  sequel = scanner.scan(/.{1,20}/)
67
67
  erroneous += sequel unless sequel.nil?
68
68
  raise ScanError.new("Unknown token #{erroneous}")
69
- end #case
70
-
69
+ end # case
71
70
 
72
- end while (token.nil? && curr_ch = scanner.getch())
71
+ break unless token.nil? && (curr_ch = scanner.getch)
72
+ end
73
73
 
74
74
  return token
75
75
  end
76
76
 
77
-
78
77
  def skip_whitespaces()
79
78
  matched = scanner.scan(/[ \t\f\n\r]+/)
80
79
  return if matched.nil?
@@ -84,10 +83,8 @@ private
84
83
  newline_detected(newline_count)
85
84
  end
86
85
 
87
-
88
86
  def newline_detected(count)
89
87
  @lineno += count
90
- @line_start = scanner.pos()
88
+ @line_start = scanner.pos
91
89
  end
92
-
93
- end # class
90
+ end # class
@@ -14,11 +14,11 @@ class CalcParser < Rley::Parser::GFGEarleyParser
14
14
  end
15
15
 
16
16
  def parse_expression(aText)
17
- lexer = CalcLexer.new(aText, self.grammar)
17
+ lexer = CalcLexer.new(aText, grammar)
18
18
  result = parse(lexer.tokens)
19
19
 
20
20
  return result
21
21
  end
22
22
  end # class
23
23
 
24
- # End of file
24
+ # End of file
@@ -3,7 +3,7 @@
3
3
 
4
4
  module Rley # Module used as a namespace
5
5
  # The version number of the gem.
6
- Version = '0.4.06'.freeze
6
+ Version = '0.4.07'.freeze
7
7
 
8
8
  # Brief description of the gem.
9
9
  Description = "Ruby implementation of the Earley's parsing algorithm".freeze
@@ -13,14 +13,19 @@ module Rley # This module is used as a namespace
13
13
  # Allowed string values are: 'first', 'last', 'first_and_last', 'other'
14
14
  attr_reader(:ranks)
15
15
 
16
+ # @return [String] The character pattern used for rendering
17
+ # a parent - child nesting
16
18
  attr_reader(:nesting_prefix)
17
19
 
20
+ # @return [String] The character pattern used for a blank indentation
18
21
  attr_reader(:blank_indent)
19
22
 
23
+ # @return [String] The character pattern for indentation and nesting
24
+ # continuation.
20
25
  attr_reader(:continuation_indent)
21
26
 
22
27
  # Constructor.
23
- # @param anIO [IO] The output stream to which the rendered grammar
28
+ # @param anIO [IO] The output stream to which the parse tree
24
29
  # is written.
25
30
  def initialize(anIO)
26
31
  super(anIO)
@@ -32,18 +37,16 @@ module Rley # This module is used as a namespace
32
37
  @continuation_indent = '| '
33
38
  end
34
39
 
35
-
36
40
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
37
41
  # Notification of a visit event: the visitor is about to visit
38
42
  # the children of a non-terminal node
39
43
  # @param parent [NonTerminalNode]
40
- # @param children [Array] array of children nodes
41
- def before_subnodes(parent, children)
44
+ # @param children [Array<ParseTreeNode>] array of children nodes
45
+ def before_subnodes(parent, _children)
42
46
  rank_of(parent)
43
47
  curr_path << parent
44
48
  end
45
49
 
46
-
47
50
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
48
51
  # Notification of a visit event: the visitor is about to visit
49
52
  # a non-terminal node
@@ -52,7 +55,6 @@ module Rley # This module is used as a namespace
52
55
  emit(aNonTerm)
53
56
  end
54
57
 
55
-
56
58
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
57
59
  # Notification of a visit event: the visitor is about to visit
58
60
  # a terminal node
@@ -61,7 +63,6 @@ module Rley # This module is used as a namespace
61
63
  emit(aTerm, ": '#{aTerm.token.lexeme}'")
62
64
  end
63
65
 
64
-
65
66
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
66
67
  # Notification of a visit event: the visitor completed the visit of
67
68
  # the children of a non-terminal node.
@@ -86,13 +87,13 @@ module Rley # This module is used as a namespace
86
87
  siblings = parent.subnodes
87
88
  siblings_last_index = siblings.size - 1
88
89
  rank = case siblings.find_index(aChild)
89
- when 0 then 'first'
90
- when siblings_last_index then 'last'
91
- else
92
- 'other'
90
+ when 0 then 'first'
91
+ when siblings_last_index then 'last'
92
+ else
93
+ 'other'
93
94
  end
94
95
  end
95
- self.ranks << rank
96
+ ranks << rank
96
97
  end
97
98
 
98
99
  # 'root', 'first', 'first_and_last', 'last', 'other'
@@ -101,7 +102,7 @@ module Rley # This module is used as a namespace
101
102
 
102
103
  prefix = ''
103
104
  @ranks.each_with_index do |rank, i|
104
- next if i == 0
105
+ next if i.zero?
105
106
 
106
107
  case rank
107
108
  when 'first', 'other'
@@ -109,7 +110,6 @@ module Rley # This module is used as a namespace
109
110
 
110
111
  when 'last', 'first_and_last', 'root'
111
112
  prefix << blank_indent
112
- else
113
113
  end
114
114
  end
115
115
 
@@ -117,7 +117,6 @@ module Rley # This module is used as a namespace
117
117
  return prefix
118
118
  end
119
119
 
120
-
121
120
  def emit(aNode, aSuffix = '')
122
121
  output.puts("#{path_prefix}#{aNode.symbol.name}#{aSuffix}")
123
122
  end
@@ -125,4 +124,4 @@ module Rley # This module is used as a namespace
125
124
  end # module
126
125
  end # module
127
126
 
128
- # End of file
127
+ # End of file
@@ -11,7 +11,6 @@ module Rley # This module is used as a namespace
11
11
  # For Ruby developers, there is RSyntaxTree by Yoichiro Hasebe.
12
12
  # (accessible via: http://yohasebe.com/rsyntaxtree/)
13
13
  class BracketNotation < BaseFormatter
14
-
15
14
  # Constructor.
16
15
  # @param anIO [IO] The output stream to which the rendered grammar
17
16
  # is written.
@@ -19,7 +18,6 @@ module Rley # This module is used as a namespace
19
18
  super(anIO)
20
19
  end
21
20
 
22
-
23
21
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
24
22
  # Notification of a visit event: the visitor is about to visit
25
23
  # a non-terminal node
@@ -28,7 +26,6 @@ module Rley # This module is used as a namespace
28
26
  write("[#{aNonTerm.symbol.name} ")
29
27
  end
30
28
 
31
-
32
29
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
33
30
  # Notification of a visit event: the visitor is about to visit
34
31
  # a terminal node
@@ -43,8 +40,8 @@ module Rley # This module is used as a namespace
43
40
  # @param aTerm [TerminalNode]
44
41
  def after_terminal(aTerm)
45
42
  # Escape all opening and closing square brackets
46
- escape_lbrackets = aTerm.token.lexeme.gsub(/\[/, "\\[")
47
- escaped = escape_lbrackets.gsub(/\]/, "\\]")
43
+ escape_lbrackets = aTerm.token.lexeme.gsub(/\[/, '\[')
44
+ escaped = escape_lbrackets.gsub(/\]/, '\]')
48
45
  write(escaped + ']')
49
46
  end
50
47
 
@@ -65,4 +62,4 @@ module Rley # This module is used as a namespace
65
62
  end # module
66
63
  end # module
67
64
 
68
- # End of file
65
+ # End of file
@@ -81,7 +81,6 @@ module Rley # This module is used as a namespace
81
81
  output_event(__method__, indentation)
82
82
  end
83
83
 
84
-
85
84
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
86
85
  # Notification of a visit event: the visitor completed the visit
87
86
  # of the given parse tree
@@ -68,7 +68,6 @@ module Rley # This module is used as a namespace
68
68
  sibling_flags[-1] = true
69
69
  end
70
70
 
71
-
72
71
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
73
72
  # Notification of a visit event: the visitor completed the visit of
74
73
  # the children of a non-terminal node.
@@ -54,12 +54,12 @@ module Rley # This module is used as a namespace
54
54
  end
55
55
 
56
56
  def done?
57
- self.to_visit.empty?
57
+ to_visit.empty?
58
58
  end
59
59
 
60
60
  def next_edge
61
- next_one = self.to_visit.shift
62
- self.visited << next_one.successor unless next_one.nil?
61
+ next_one = to_visit.shift
62
+ visited << next_one.successor unless next_one.nil?
63
63
 
64
64
  return next_one
65
65
  end
@@ -67,9 +67,11 @@ module Rley # This module is used as a namespace
67
67
 
68
68
  # Walk over all the vertices of the graph that are reachable from a given
69
69
  # start vertex. This is a depth-first graph traversal.
70
- # @param aStartVertex [StartVertex] the depth-first traversal begins from here
71
- # @param visitAction [Proc] block code called when a new graph vertex is found
72
- def traverse_df(aStartVertex, &visitAction)
70
+ # @param aStartVertex [StartVertex] the depth-first traversal begins
71
+ # from here
72
+ # @param visitAction [Proc] block code called when a new graph vertex
73
+ # is found
74
+ def traverse_df(aStartVertex, &_visitAction)
73
75
  visited = Set.new
74
76
  stack = []
75
77
  visitee = aStartVertex
@@ -77,7 +79,7 @@ module Rley # This module is used as a namespace
77
79
  begin
78
80
  first_time = !visited.include?(visitee)
79
81
  if first_time
80
- visitAction.call(visitee)
82
+ yield(visitee)
81
83
  visited << visitee
82
84
  end
83
85
 
@@ -104,7 +106,7 @@ module Rley # This module is used as a namespace
104
106
  visitee = curr_edge.successor unless curr_edge.nil?
105
107
  end until stack.empty?
106
108
  # Now process the end vertex matching the initial start vertex
107
- visitAction.call(end_vertex_for[aStartVertex.non_terminal])
109
+ yield(end_vertex_for[aStartVertex.non_terminal])
108
110
  end
109
111
 
110
112
  private
@@ -165,7 +167,6 @@ module Rley # This module is used as a namespace
165
167
  add_vertex(new_end_vertex)
166
168
  end
167
169
 
168
-
169
170
  def add_single_item(aDottedItem)
170
171
  new_vertex = ItemVertex.new(aDottedItem)
171
172
  add_vertex(new_vertex)
@@ -265,7 +266,6 @@ module Rley # This module is used as a namespace
265
266
  ShortcutEdge.new(fromVertex, toVertex)
266
267
  end
267
268
 
268
-
269
269
  # Retrieve the return edge that matches the given
270
270
  # call edge.
271
271
  def get_matching_return(aCallEdge)
@@ -277,6 +277,7 @@ module Rley # This module is used as a namespace
277
277
 
278
278
  # Retrieve the return edge with specified key
279
279
  return_edge = end_vertex.edges.find { |edge| edge.key == ret_key }
280
+ return return_edge
280
281
  end
281
282
 
282
283
  # Mark non-terminal symbols that cannot be derived from the start symbol.
@@ -40,14 +40,12 @@ module Rley # This module is used as a namespace
40
40
  pforest.accept(self)
41
41
  end
42
42
 
43
-
44
43
  # Visit event. The visitor is about to visit the pforest.
45
44
  # @param aParseForest [ParseForest] the pforest to visit.
46
45
  def start_visit_pforest(aParseForest)
47
46
  broadcast(:before_pforest, aParseForest)
48
47
  end
49
48
 
50
-
51
49
  # Visit event. The visitor is about to visit the given non terminal node.
52
50
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
53
51
  def visit_nonterminal(aNonTerminalNode)
@@ -69,7 +67,6 @@ module Rley # This module is used as a namespace
69
67
  broadcast(:after_terminal, aTerminalNode)
70
68
  end
71
69
 
72
-
73
70
  # Visit event. The visitor has completed its visit of the given
74
71
  # non-terminal node.
75
72
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
@@ -37,14 +37,12 @@ module Rley # This module is used as a namespace
37
37
  ptree.accept(self)
38
38
  end
39
39
 
40
-
41
40
  # Visit event. The visitor is about to visit the ptree.
42
41
  # @param aParseTree [ParseTree] the ptree to visit.
43
42
  def start_visit_ptree(aParseTree)
44
43
  broadcast(:before_ptree, aParseTree)
45
44
  end
46
45
 
47
-
48
46
  # Visit event. The visitor is about to visit the given non terminal node.
49
47
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
50
48
  def visit_nonterminal(aNonTerminalNode)
@@ -66,7 +64,6 @@ module Rley # This module is used as a namespace
66
64
  broadcast(:after_terminal, aTerminalNode)
67
65
  end
68
66
 
69
-
70
67
  # Visit event. The visitor has completed its visit of the given
71
68
  # non-terminal node.
72
69
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
@@ -16,7 +16,7 @@ module Rley # Module used as a namespace
16
16
 
17
17
  # Returns the result of invoking reason.to_s.
18
18
  def message()
19
- return self.to_s
19
+ return to_s
20
20
  end
21
21
 
22
22
  # Return this reason's class name and message
@@ -68,10 +68,8 @@ module Rley # Module used as a namespace
68
68
  end
69
69
  return explain
70
70
  end
71
-
72
71
  end # class
73
72
 
74
-
75
73
  # This parse error occurs when the current token from the input
76
74
  # is unexpected according to the grammar rules.
77
75
  class UnexpectedToken < ExpectationNotMet
@@ -86,7 +84,6 @@ module Rley # Module used as a namespace
86
84
  end
87
85
  end # class
88
86
 
89
-
90
87
  # This parse error occurs when all input tokens were consumed
91
88
  # but the parser still expected one or more tokens from the input.
92
89
  class PrematureInputEnd < ExpectationNotMet
@@ -103,4 +100,3 @@ module Rley # Module used as a namespace
103
100
  end # module
104
101
 
105
102
  # End of file
106
-
@@ -12,7 +12,7 @@ module Rley # This module is used as a namespace
12
12
  # An array of entry sets (one per input token + 1)
13
13
  attr_reader(:sets)
14
14
 
15
- # @param tokenCount [Fixnum] The number of lexemes in the input to parse.
15
+ # @param tokenCount [Integer] The number of lexemes in the input to parse.
16
16
  def initialize(tokenCount, aGFGraph)
17
17
  @sets = Array.new(tokenCount + 1) { |_| ParseEntrySet.new }
18
18
  push_entry(aGFGraph.start_vertex, 0, 0, :start_rule)
@@ -41,7 +41,7 @@ module Rley # This module is used as a namespace
41
41
  end
42
42
 
43
43
  # Push a parse entry for the chart entry with given index
44
- def push_entry(aVertex, anOrigin, anIndex, aReason)
44
+ def push_entry(aVertex, anOrigin, anIndex, _reason)
45
45
  new_entry = ParseEntry.new(aVertex, anOrigin)
46
46
  pushed = self[anIndex].push_entry(new_entry)
47
47
 
@@ -21,7 +21,7 @@ module Rley # This module is used as a namespace
21
21
  def parse(aTokenSequence)
22
22
  result = GFGParsing.new(gf_graph, aTokenSequence)
23
23
  last_token_index = aTokenSequence.size
24
- if last_token_index == 0 && !grammar.start_symbol.nullable?
24
+ if last_token_index.zero? && !grammar.start_symbol.nullable?
25
25
  return unexpected_empty_input(result)
26
26
  end
27
27
 
@@ -104,7 +104,6 @@ module Rley # This module is used as a namespace
104
104
  aParsing.faulty(NoInput.new)
105
105
  return aParsing
106
106
  end
107
-
108
107
  end # class
109
108
  end # module
110
109
  end # module