rley 0.4.06 → 0.4.07

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/.rubocop.yml +22 -13
  3. data/.travis.yml +8 -7
  4. data/CHANGELOG.md +9 -0
  5. data/README.md +9 -10
  6. data/examples/NLP/mini_en_demo.rb +7 -7
  7. data/examples/data_formats/JSON/cli_options.rb +5 -6
  8. data/examples/data_formats/JSON/{JSON_demo.rb → json_demo.rb} +2 -2
  9. data/examples/data_formats/JSON/{JSON_grammar.rb → json_grammar.rb} +10 -10
  10. data/examples/data_formats/JSON/{JSON_lexer.rb → json_lexer.rb} +17 -22
  11. data/examples/data_formats/JSON/{JSON_parser.rb → json_parser.rb} +2 -2
  12. data/examples/general/calc/calc_demo.rb +1 -1
  13. data/examples/general/calc/calc_grammar.rb +5 -5
  14. data/examples/general/calc/calc_lexer.rb +14 -17
  15. data/examples/general/calc/calc_parser.rb +2 -2
  16. data/lib/rley/constants.rb +1 -1
  17. data/lib/rley/formatter/asciitree.rb +15 -16
  18. data/lib/rley/formatter/bracket_notation.rb +3 -6
  19. data/lib/rley/formatter/debug.rb +0 -1
  20. data/lib/rley/formatter/json.rb +0 -1
  21. data/lib/rley/gfg/grm_flow_graph.rb +11 -10
  22. data/lib/rley/parse_forest_visitor.rb +0 -3
  23. data/lib/rley/parse_tree_visitor.rb +0 -3
  24. data/lib/rley/parser/error_reason.rb +1 -5
  25. data/lib/rley/parser/gfg_chart.rb +2 -2
  26. data/lib/rley/parser/gfg_earley_parser.rb +1 -2
  27. data/lib/rley/parser/gfg_parsing.rb +3 -7
  28. data/lib/rley/parser/parse_entry.rb +0 -1
  29. data/lib/rley/parser/parse_entry_set.rb +15 -16
  30. data/lib/rley/parser/parse_forest_builder.rb +8 -23
  31. data/lib/rley/parser/parse_state.rb +1 -1
  32. data/lib/rley/parser/parse_tree_builder.rb +2 -30
  33. data/lib/rley/parser/parse_tree_factory.rb +1 -1
  34. data/lib/rley/parser/parse_walker_factory.rb +3 -6
  35. data/lib/rley/parser/state_set.rb +0 -1
  36. data/lib/rley/ptree/parse_tree.rb +0 -1
  37. data/lib/rley/ptree/terminal_node.rb +4 -1
  38. data/lib/rley/rley_error.rb +1 -1
  39. data/lib/rley/sppf/composite_node.rb +0 -1
  40. data/lib/rley/sppf/parse_forest.rb +0 -1
  41. data/lib/rley/syntax/grammar.rb +5 -9
  42. data/lib/rley/syntax/grammar_builder.rb +8 -11
  43. data/lib/rley/syntax/grm_symbol.rb +0 -1
  44. data/lib/rley/syntax/production.rb +5 -4
  45. data/lib/rley/tokens/token_range.rb +0 -1
  46. data/spec/rley/formatter/bracket_notation_spec.rb +3 -1
  47. data/spec/rley/gfg/grm_flow_graph_spec.rb +15 -46
  48. data/spec/rley/gfg/item_vertex_spec.rb +1 -1
  49. data/spec/rley/parse_forest_visitor_spec.rb +1 -1
  50. data/spec/rley/parse_tree_visitor_spec.rb +2 -2
  51. data/spec/rley/parser/error_reason_spec.rb +19 -14
  52. data/spec/rley/parser/gfg_chart_spec.rb +1 -1
  53. data/spec/rley/parser/gfg_earley_parser_spec.rb +15 -15
  54. data/spec/rley/parser/gfg_parsing_spec.rb +3 -3
  55. data/spec/rley/parser/groucho_spec.rb +6 -7
  56. data/spec/rley/parser/parse_forest_builder_spec.rb +5 -5
  57. data/spec/rley/parser/parse_forest_factory_spec.rb +5 -5
  58. data/spec/rley/parser/parse_state_spec.rb +8 -0
  59. data/spec/rley/parser/parse_tracer_spec.rb +1 -1
  60. data/spec/rley/parser/parse_tree_builder_spec.rb +26 -29
  61. data/spec/rley/parser/parse_tree_factory_spec.rb +5 -5
  62. data/spec/rley/parser/parse_walker_factory_spec.rb +5 -5
  63. data/spec/rley/ptree/parse_tree_node_spec.rb +2 -2
  64. data/spec/rley/ptree/terminal_node_spec.rb +1 -1
  65. data/spec/rley/support/ambiguous_grammar_helper.rb +1 -1
  66. data/spec/rley/support/expectation_helper.rb +0 -1
  67. data/spec/rley/support/grammar_abc_helper.rb +1 -1
  68. data/spec/rley/support/grammar_ambig01_helper.rb +2 -3
  69. data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
  70. data/spec/rley/support/grammar_l0_helper.rb +7 -8
  71. data/spec/rley/support/grammar_pb_helper.rb +3 -4
  72. data/spec/rley/support/grammar_sppf_helper.rb +4 -4
  73. data/spec/rley/syntax/grammar_builder_spec.rb +5 -4
  74. data/spec/rley/syntax/grammar_spec.rb +10 -11
  75. data/spec/rley/syntax/symbol_seq_spec.rb +2 -2
  76. data/spec/rley/syntax/terminal_spec.rb +1 -1
  77. metadata +31 -31
@@ -1,7 +1,7 @@
1
1
  # File: calc_lexer.rb
2
2
  # Lexer for a basic arithmetical expression parser
3
3
  require 'strscan'
4
- require 'rley' # Load the gem
4
+ require 'rley' # Load the gem
5
5
 
6
6
 
7
7
  class CalcLexer
@@ -16,21 +16,20 @@ class CalcLexer
16
16
  '+' => 'PLUS',
17
17
  '-' => 'MINUS',
18
18
  '*' => 'STAR',
19
- '/' => 'DIVIDE',
20
- }
19
+ '/' => 'DIVIDE'
20
+ }.freeze
21
21
 
22
- class ScanError < StandardError ; end
22
+ class ScanError < StandardError; end
23
23
 
24
- public
25
24
  def initialize(source, aGrammar)
26
25
  @scanner = StringScanner.new(source)
27
26
  @name2symbol = aGrammar.name2symbol
28
- @lineno = 1
27
+ @lineno = 1
29
28
  end
30
29
 
31
30
  def tokens()
32
31
  tok_sequence = []
33
- until @scanner.eos? do
32
+ until @scanner.eos?
34
33
  token = _next_token
35
34
  tok_sequence << token unless token.nil?
36
35
  end
@@ -38,13 +37,14 @@ public
38
37
  return tok_sequence
39
38
  end
40
39
 
41
- private
40
+ private
41
+
42
42
  def _next_token()
43
43
  token = nil
44
44
  skip_whitespaces
45
45
  curr_ch = scanner.getch # curr_ch is at start of token or eof reached...
46
46
 
47
- begin
47
+ loop do
48
48
  break if curr_ch.nil?
49
49
 
50
50
  case curr_ch
@@ -66,15 +66,14 @@ private
66
66
  sequel = scanner.scan(/.{1,20}/)
67
67
  erroneous += sequel unless sequel.nil?
68
68
  raise ScanError.new("Unknown token #{erroneous}")
69
- end #case
70
-
69
+ end # case
71
70
 
72
- end while (token.nil? && curr_ch = scanner.getch())
71
+ break unless token.nil? && (curr_ch = scanner.getch)
72
+ end
73
73
 
74
74
  return token
75
75
  end
76
76
 
77
-
78
77
  def skip_whitespaces()
79
78
  matched = scanner.scan(/[ \t\f\n\r]+/)
80
79
  return if matched.nil?
@@ -84,10 +83,8 @@ private
84
83
  newline_detected(newline_count)
85
84
  end
86
85
 
87
-
88
86
  def newline_detected(count)
89
87
  @lineno += count
90
- @line_start = scanner.pos()
88
+ @line_start = scanner.pos
91
89
  end
92
-
93
- end # class
90
+ end # class
@@ -14,11 +14,11 @@ class CalcParser < Rley::Parser::GFGEarleyParser
14
14
  end
15
15
 
16
16
  def parse_expression(aText)
17
- lexer = CalcLexer.new(aText, self.grammar)
17
+ lexer = CalcLexer.new(aText, grammar)
18
18
  result = parse(lexer.tokens)
19
19
 
20
20
  return result
21
21
  end
22
22
  end # class
23
23
 
24
- # End of file
24
+ # End of file
@@ -3,7 +3,7 @@
3
3
 
4
4
  module Rley # Module used as a namespace
5
5
  # The version number of the gem.
6
- Version = '0.4.06'.freeze
6
+ Version = '0.4.07'.freeze
7
7
 
8
8
  # Brief description of the gem.
9
9
  Description = "Ruby implementation of the Earley's parsing algorithm".freeze
@@ -13,14 +13,19 @@ module Rley # This module is used as a namespace
13
13
  # Allowed string values are: 'first', 'last', 'first_and_last', 'other'
14
14
  attr_reader(:ranks)
15
15
 
16
+ # @return [String] The character pattern used for rendering
17
+ # a parent - child nesting
16
18
  attr_reader(:nesting_prefix)
17
19
 
20
+ # @return [String] The character pattern used for a blank indentation
18
21
  attr_reader(:blank_indent)
19
22
 
23
+ # @return [String] The character pattern for indentation and nesting
24
+ # continuation.
20
25
  attr_reader(:continuation_indent)
21
26
 
22
27
  # Constructor.
23
- # @param anIO [IO] The output stream to which the rendered grammar
28
+ # @param anIO [IO] The output stream to which the parse tree
24
29
  # is written.
25
30
  def initialize(anIO)
26
31
  super(anIO)
@@ -32,18 +37,16 @@ module Rley # This module is used as a namespace
32
37
  @continuation_indent = '| '
33
38
  end
34
39
 
35
-
36
40
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
37
41
  # Notification of a visit event: the visitor is about to visit
38
42
  # the children of a non-terminal node
39
43
  # @param parent [NonTerminalNode]
40
- # @param children [Array] array of children nodes
41
- def before_subnodes(parent, children)
44
+ # @param children [Array<ParseTreeNode>] array of children nodes
45
+ def before_subnodes(parent, _children)
42
46
  rank_of(parent)
43
47
  curr_path << parent
44
48
  end
45
49
 
46
-
47
50
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
48
51
  # Notification of a visit event: the visitor is about to visit
49
52
  # a non-terminal node
@@ -52,7 +55,6 @@ module Rley # This module is used as a namespace
52
55
  emit(aNonTerm)
53
56
  end
54
57
 
55
-
56
58
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
57
59
  # Notification of a visit event: the visitor is about to visit
58
60
  # a terminal node
@@ -61,7 +63,6 @@ module Rley # This module is used as a namespace
61
63
  emit(aTerm, ": '#{aTerm.token.lexeme}'")
62
64
  end
63
65
 
64
-
65
66
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
66
67
  # Notification of a visit event: the visitor completed the visit of
67
68
  # the children of a non-terminal node.
@@ -86,13 +87,13 @@ module Rley # This module is used as a namespace
86
87
  siblings = parent.subnodes
87
88
  siblings_last_index = siblings.size - 1
88
89
  rank = case siblings.find_index(aChild)
89
- when 0 then 'first'
90
- when siblings_last_index then 'last'
91
- else
92
- 'other'
90
+ when 0 then 'first'
91
+ when siblings_last_index then 'last'
92
+ else
93
+ 'other'
93
94
  end
94
95
  end
95
- self.ranks << rank
96
+ ranks << rank
96
97
  end
97
98
 
98
99
  # 'root', 'first', 'first_and_last', 'last', 'other'
@@ -101,7 +102,7 @@ module Rley # This module is used as a namespace
101
102
 
102
103
  prefix = ''
103
104
  @ranks.each_with_index do |rank, i|
104
- next if i == 0
105
+ next if i.zero?
105
106
 
106
107
  case rank
107
108
  when 'first', 'other'
@@ -109,7 +110,6 @@ module Rley # This module is used as a namespace
109
110
 
110
111
  when 'last', 'first_and_last', 'root'
111
112
  prefix << blank_indent
112
- else
113
113
  end
114
114
  end
115
115
 
@@ -117,7 +117,6 @@ module Rley # This module is used as a namespace
117
117
  return prefix
118
118
  end
119
119
 
120
-
121
120
  def emit(aNode, aSuffix = '')
122
121
  output.puts("#{path_prefix}#{aNode.symbol.name}#{aSuffix}")
123
122
  end
@@ -125,4 +124,4 @@ module Rley # This module is used as a namespace
125
124
  end # module
126
125
  end # module
127
126
 
128
- # End of file
127
+ # End of file
@@ -11,7 +11,6 @@ module Rley # This module is used as a namespace
11
11
  # For Ruby developers, there is RSyntaxTree by Yoichiro Hasebe.
12
12
  # (accessible via: http://yohasebe.com/rsyntaxtree/)
13
13
  class BracketNotation < BaseFormatter
14
-
15
14
  # Constructor.
16
15
  # @param anIO [IO] The output stream to which the rendered grammar
17
16
  # is written.
@@ -19,7 +18,6 @@ module Rley # This module is used as a namespace
19
18
  super(anIO)
20
19
  end
21
20
 
22
-
23
21
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
24
22
  # Notification of a visit event: the visitor is about to visit
25
23
  # a non-terminal node
@@ -28,7 +26,6 @@ module Rley # This module is used as a namespace
28
26
  write("[#{aNonTerm.symbol.name} ")
29
27
  end
30
28
 
31
-
32
29
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
33
30
  # Notification of a visit event: the visitor is about to visit
34
31
  # a terminal node
@@ -43,8 +40,8 @@ module Rley # This module is used as a namespace
43
40
  # @param aTerm [TerminalNode]
44
41
  def after_terminal(aTerm)
45
42
  # Escape all opening and closing square brackets
46
- escape_lbrackets = aTerm.token.lexeme.gsub(/\[/, "\\[")
47
- escaped = escape_lbrackets.gsub(/\]/, "\\]")
43
+ escape_lbrackets = aTerm.token.lexeme.gsub(/\[/, '\[')
44
+ escaped = escape_lbrackets.gsub(/\]/, '\]')
48
45
  write(escaped + ']')
49
46
  end
50
47
 
@@ -65,4 +62,4 @@ module Rley # This module is used as a namespace
65
62
  end # module
66
63
  end # module
67
64
 
68
- # End of file
65
+ # End of file
@@ -81,7 +81,6 @@ module Rley # This module is used as a namespace
81
81
  output_event(__method__, indentation)
82
82
  end
83
83
 
84
-
85
84
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
86
85
  # Notification of a visit event: the visitor completed the visit
87
86
  # of the given parse tree
@@ -68,7 +68,6 @@ module Rley # This module is used as a namespace
68
68
  sibling_flags[-1] = true
69
69
  end
70
70
 
71
-
72
71
  # Method called by a ParseTreeVisitor to which the formatter subscribed.
73
72
  # Notification of a visit event: the visitor completed the visit of
74
73
  # the children of a non-terminal node.
@@ -54,12 +54,12 @@ module Rley # This module is used as a namespace
54
54
  end
55
55
 
56
56
  def done?
57
- self.to_visit.empty?
57
+ to_visit.empty?
58
58
  end
59
59
 
60
60
  def next_edge
61
- next_one = self.to_visit.shift
62
- self.visited << next_one.successor unless next_one.nil?
61
+ next_one = to_visit.shift
62
+ visited << next_one.successor unless next_one.nil?
63
63
 
64
64
  return next_one
65
65
  end
@@ -67,9 +67,11 @@ module Rley # This module is used as a namespace
67
67
 
68
68
  # Walk over all the vertices of the graph that are reachable from a given
69
69
  # start vertex. This is a depth-first graph traversal.
70
- # @param aStartVertex [StartVertex] the depth-first traversal begins from here
71
- # @param visitAction [Proc] block code called when a new graph vertex is found
72
- def traverse_df(aStartVertex, &visitAction)
70
+ # @param aStartVertex [StartVertex] the depth-first traversal begins
71
+ # from here
72
+ # @param visitAction [Proc] block code called when a new graph vertex
73
+ # is found
74
+ def traverse_df(aStartVertex, &_visitAction)
73
75
  visited = Set.new
74
76
  stack = []
75
77
  visitee = aStartVertex
@@ -77,7 +79,7 @@ module Rley # This module is used as a namespace
77
79
  begin
78
80
  first_time = !visited.include?(visitee)
79
81
  if first_time
80
- visitAction.call(visitee)
82
+ yield(visitee)
81
83
  visited << visitee
82
84
  end
83
85
 
@@ -104,7 +106,7 @@ module Rley # This module is used as a namespace
104
106
  visitee = curr_edge.successor unless curr_edge.nil?
105
107
  end until stack.empty?
106
108
  # Now process the end vertex matching the initial start vertex
107
- visitAction.call(end_vertex_for[aStartVertex.non_terminal])
109
+ yield(end_vertex_for[aStartVertex.non_terminal])
108
110
  end
109
111
 
110
112
  private
@@ -165,7 +167,6 @@ module Rley # This module is used as a namespace
165
167
  add_vertex(new_end_vertex)
166
168
  end
167
169
 
168
-
169
170
  def add_single_item(aDottedItem)
170
171
  new_vertex = ItemVertex.new(aDottedItem)
171
172
  add_vertex(new_vertex)
@@ -265,7 +266,6 @@ module Rley # This module is used as a namespace
265
266
  ShortcutEdge.new(fromVertex, toVertex)
266
267
  end
267
268
 
268
-
269
269
  # Retrieve the return edge that matches the given
270
270
  # call edge.
271
271
  def get_matching_return(aCallEdge)
@@ -277,6 +277,7 @@ module Rley # This module is used as a namespace
277
277
 
278
278
  # Retrieve the return edge with specified key
279
279
  return_edge = end_vertex.edges.find { |edge| edge.key == ret_key }
280
+ return return_edge
280
281
  end
281
282
 
282
283
  # Mark non-terminal symbols that cannot be derived from the start symbol.
@@ -40,14 +40,12 @@ module Rley # This module is used as a namespace
40
40
  pforest.accept(self)
41
41
  end
42
42
 
43
-
44
43
  # Visit event. The visitor is about to visit the pforest.
45
44
  # @param aParseForest [ParseForest] the pforest to visit.
46
45
  def start_visit_pforest(aParseForest)
47
46
  broadcast(:before_pforest, aParseForest)
48
47
  end
49
48
 
50
-
51
49
  # Visit event. The visitor is about to visit the given non terminal node.
52
50
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
53
51
  def visit_nonterminal(aNonTerminalNode)
@@ -69,7 +67,6 @@ module Rley # This module is used as a namespace
69
67
  broadcast(:after_terminal, aTerminalNode)
70
68
  end
71
69
 
72
-
73
70
  # Visit event. The visitor has completed its visit of the given
74
71
  # non-terminal node.
75
72
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
@@ -37,14 +37,12 @@ module Rley # This module is used as a namespace
37
37
  ptree.accept(self)
38
38
  end
39
39
 
40
-
41
40
  # Visit event. The visitor is about to visit the ptree.
42
41
  # @param aParseTree [ParseTree] the ptree to visit.
43
42
  def start_visit_ptree(aParseTree)
44
43
  broadcast(:before_ptree, aParseTree)
45
44
  end
46
45
 
47
-
48
46
  # Visit event. The visitor is about to visit the given non terminal node.
49
47
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
50
48
  def visit_nonterminal(aNonTerminalNode)
@@ -66,7 +64,6 @@ module Rley # This module is used as a namespace
66
64
  broadcast(:after_terminal, aTerminalNode)
67
65
  end
68
66
 
69
-
70
67
  # Visit event. The visitor has completed its visit of the given
71
68
  # non-terminal node.
72
69
  # @param aNonTerminalNode [NonTerminalNode] the node to visit.
@@ -16,7 +16,7 @@ module Rley # Module used as a namespace
16
16
 
17
17
  # Returns the result of invoking reason.to_s.
18
18
  def message()
19
- return self.to_s
19
+ return to_s
20
20
  end
21
21
 
22
22
  # Return this reason's class name and message
@@ -68,10 +68,8 @@ module Rley # Module used as a namespace
68
68
  end
69
69
  return explain
70
70
  end
71
-
72
71
  end # class
73
72
 
74
-
75
73
  # This parse error occurs when the current token from the input
76
74
  # is unexpected according to the grammar rules.
77
75
  class UnexpectedToken < ExpectationNotMet
@@ -86,7 +84,6 @@ module Rley # Module used as a namespace
86
84
  end
87
85
  end # class
88
86
 
89
-
90
87
  # This parse error occurs when all input tokens were consumed
91
88
  # but the parser still expected one or more tokens from the input.
92
89
  class PrematureInputEnd < ExpectationNotMet
@@ -103,4 +100,3 @@ module Rley # Module used as a namespace
103
100
  end # module
104
101
 
105
102
  # End of file
106
-
@@ -12,7 +12,7 @@ module Rley # This module is used as a namespace
12
12
  # An array of entry sets (one per input token + 1)
13
13
  attr_reader(:sets)
14
14
 
15
- # @param tokenCount [Fixnum] The number of lexemes in the input to parse.
15
+ # @param tokenCount [Integer] The number of lexemes in the input to parse.
16
16
  def initialize(tokenCount, aGFGraph)
17
17
  @sets = Array.new(tokenCount + 1) { |_| ParseEntrySet.new }
18
18
  push_entry(aGFGraph.start_vertex, 0, 0, :start_rule)
@@ -41,7 +41,7 @@ module Rley # This module is used as a namespace
41
41
  end
42
42
 
43
43
  # Push a parse entry for the chart entry with given index
44
- def push_entry(aVertex, anOrigin, anIndex, aReason)
44
+ def push_entry(aVertex, anOrigin, anIndex, _reason)
45
45
  new_entry = ParseEntry.new(aVertex, anOrigin)
46
46
  pushed = self[anIndex].push_entry(new_entry)
47
47
 
@@ -21,7 +21,7 @@ module Rley # This module is used as a namespace
21
21
  def parse(aTokenSequence)
22
22
  result = GFGParsing.new(gf_graph, aTokenSequence)
23
23
  last_token_index = aTokenSequence.size
24
- if last_token_index == 0 && !grammar.start_symbol.nullable?
24
+ if last_token_index.zero? && !grammar.start_symbol.nullable?
25
25
  return unexpected_empty_input(result)
26
26
  end
27
27
 
@@ -104,7 +104,6 @@ module Rley # This module is used as a namespace
104
104
  aParsing.faulty(NoInput.new)
105
105
  return aParsing
106
106
  end
107
-
108
107
  end # class
109
108
  end # module
110
109
  end # module