rley 0.7.07 → 0.7.08
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +348 -54
- data/LICENSE.txt +1 -1
- data/README.md +3 -2
- data/examples/NLP/engtagger.rb +193 -190
- data/examples/NLP/nano_eng/nano_grammar.rb +5 -5
- data/examples/data_formats/JSON/cli_options.rb +1 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +12 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +12 -21
- data/examples/data_formats/JSON/json_grammar.rb +2 -2
- data/examples/data_formats/JSON/json_lexer.rb +8 -8
- data/examples/data_formats/JSON/json_minifier.rb +1 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +13 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +23 -37
- data/examples/general/calc_iter1/calc_grammar.rb +2 -2
- data/examples/general/calc_iter1/calc_lexer.rb +6 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +5 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +5 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +27 -43
- data/examples/general/calc_iter2/calc_grammar.rb +3 -3
- data/examples/general/calc_iter2/calc_lexer.rb +11 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +26 -26
- data/examples/general/left.rb +2 -2
- data/examples/general/right.rb +2 -2
- data/lib/rley/base/dotted_item.rb +23 -31
- data/lib/rley/constants.rb +2 -2
- data/lib/rley/engine.rb +20 -23
- data/lib/rley/formatter/asciitree.rb +3 -3
- data/lib/rley/formatter/bracket_notation.rb +1 -8
- data/lib/rley/formatter/debug.rb +6 -6
- data/lib/rley/formatter/json.rb +2 -2
- data/lib/rley/gfg/call_edge.rb +1 -1
- data/lib/rley/gfg/edge.rb +5 -5
- data/lib/rley/gfg/end_vertex.rb +2 -6
- data/lib/rley/gfg/epsilon_edge.rb +1 -5
- data/lib/rley/gfg/grm_flow_graph.rb +27 -23
- data/lib/rley/gfg/item_vertex.rb +10 -10
- data/lib/rley/gfg/non_terminal_vertex.rb +4 -4
- data/lib/rley/gfg/scan_edge.rb +1 -1
- data/lib/rley/gfg/shortcut_edge.rb +2 -2
- data/lib/rley/gfg/start_vertex.rb +4 -8
- data/lib/rley/gfg/vertex.rb +43 -39
- data/lib/rley/lexical/token_range.rb +6 -6
- data/lib/rley/parse_forest_visitor.rb +5 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +9 -11
- data/lib/rley/parse_rep/cst_builder.rb +5 -6
- data/lib/rley/parse_rep/parse_forest_builder.rb +20 -18
- data/lib/rley/parse_rep/parse_forest_factory.rb +3 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +11 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +4 -4
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -27
- data/lib/rley/parse_tree_visitor.rb +1 -1
- data/lib/rley/parser/error_reason.rb +4 -5
- data/lib/rley/parser/gfg_chart.rb +20 -22
- data/lib/rley/parser/gfg_parsing.rb +16 -30
- data/lib/rley/parser/parse_entry.rb +25 -31
- data/lib/rley/parser/parse_entry_set.rb +18 -15
- data/lib/rley/parser/parse_entry_tracker.rb +4 -4
- data/lib/rley/parser/parse_state.rb +16 -21
- data/lib/rley/parser/parse_state_tracker.rb +4 -4
- data/lib/rley/parser/parse_tracer.rb +13 -13
- data/lib/rley/parser/parse_walker_factory.rb +23 -28
- data/lib/rley/parser/state_set.rb +9 -10
- data/lib/rley/ptree/non_terminal_node.rb +7 -5
- data/lib/rley/ptree/parse_tree.rb +3 -3
- data/lib/rley/ptree/parse_tree_node.rb +5 -5
- data/lib/rley/ptree/terminal_node.rb +7 -7
- data/lib/rley/rley_error.rb +12 -12
- data/lib/rley/sppf/alternative_node.rb +6 -6
- data/lib/rley/sppf/composite_node.rb +7 -7
- data/lib/rley/sppf/epsilon_node.rb +3 -3
- data/lib/rley/sppf/leaf_node.rb +3 -3
- data/lib/rley/sppf/parse_forest.rb +16 -16
- data/lib/rley/sppf/sppf_node.rb +7 -8
- data/lib/rley/sppf/token_node.rb +3 -3
- data/lib/rley/syntax/grammar.rb +5 -5
- data/lib/rley/syntax/grammar_builder.rb +9 -9
- data/lib/rley/syntax/grm_symbol.rb +6 -6
- data/lib/rley/syntax/non_terminal.rb +9 -15
- data/lib/rley/syntax/production.rb +10 -10
- data/lib/rley/syntax/symbol_seq.rb +7 -9
- data/lib/rley/syntax/terminal.rb +4 -5
- data/lib/rley/syntax/verbatim_symbol.rb +3 -3
- data/lib/support/base_tokenizer.rb +19 -18
- data/spec/rley/base/dotted_item_spec.rb +2 -2
- data/spec/rley/engine_spec.rb +17 -15
- data/spec/rley/formatter/asciitree_spec.rb +7 -7
- data/spec/rley/formatter/bracket_notation_spec.rb +13 -13
- data/spec/rley/formatter/json_spec.rb +1 -1
- data/spec/rley/gfg/end_vertex_spec.rb +5 -5
- data/spec/rley/gfg/item_vertex_spec.rb +10 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +3 -3
- data/spec/rley/gfg/shortcut_edge_spec.rb +1 -1
- data/spec/rley/gfg/start_vertex_spec.rb +5 -5
- data/spec/rley/gfg/vertex_spec.rb +3 -3
- data/spec/rley/lexical/token_range_spec.rb +16 -16
- data/spec/rley/lexical/token_spec.rb +2 -2
- data/spec/rley/parse_forest_visitor_spec.rb +165 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +44 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +6 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +5 -5
- data/spec/rley/parse_rep/groucho_spec.rb +21 -21
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +26 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +6 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +2 -2
- data/spec/rley/parse_tree_visitor_spec.rb +10 -8
- data/spec/rley/parser/error_reason_spec.rb +6 -6
- data/spec/rley/parser/gfg_earley_parser_spec.rb +4 -2
- data/spec/rley/parser/gfg_parsing_spec.rb +4 -8
- data/spec/rley/parser/parse_entry_spec.rb +19 -19
- data/spec/rley/parser/parse_state_spec.rb +5 -5
- data/spec/rley/parser/parse_walker_factory_spec.rb +1 -1
- data/spec/rley/parser/state_set_spec.rb +22 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +5 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +4 -4
- data/spec/rley/ptree/terminal_node_spec.rb +6 -6
- data/spec/rley/sppf/alternative_node_spec.rb +6 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +3 -3
- data/spec/rley/sppf/token_node_spec.rb +4 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +3 -4
- data/spec/rley/support/grammar_abc_helper.rb +2 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +4 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +4 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +4 -5
- data/spec/rley/support/grammar_l0_helper.rb +10 -11
- data/spec/rley/support/grammar_pb_helper.rb +6 -5
- data/spec/rley/support/grammar_sppf_helper.rb +1 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +5 -5
- data/spec/rley/syntax/grammar_spec.rb +6 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +1 -1
- data/spec/rley/syntax/non_terminal_spec.rb +8 -8
- data/spec/rley/syntax/production_spec.rb +13 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +2 -2
- data/spec/rley/syntax/terminal_spec.rb +5 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -12
- data/spec/support/base_tokenizer_spec.rb +7 -2
- metadata +21 -62
- data/.simplecov +0 -8
@@ -18,12 +18,12 @@ module Rley # This module is used as a namespace
|
|
18
18
|
# to the matching grammar symbol object.
|
19
19
|
attr_reader(:symbols)
|
20
20
|
|
21
|
-
# @return [Array<Production>] The list of production rules for
|
21
|
+
# @return [Array<Production>] The list of production rules for
|
22
22
|
# the grammar to build.
|
23
23
|
attr_reader(:productions)
|
24
24
|
|
25
25
|
# Creates a new grammar builder.
|
26
|
-
# @param aBlock [Proc] code block used to build the grammar.
|
26
|
+
# @param aBlock [Proc] code block used to build the grammar.
|
27
27
|
# @example Building a tiny English grammar
|
28
28
|
# builder = Rley::Syntax::GrammarBuilder.new do
|
29
29
|
# add_terminals('n', 'v', 'adj', 'det')
|
@@ -45,7 +45,7 @@ module Rley # This module is used as a namespace
|
|
45
45
|
# @param aSymbolName [String] the name of a grammar symbol.
|
46
46
|
# @return [GrmSymbol] the retrieved symbol object.
|
47
47
|
def [](aSymbolName)
|
48
|
-
|
48
|
+
symbols[aSymbolName]
|
49
49
|
end
|
50
50
|
|
51
51
|
# Add the given terminal symbols to the grammar of the language
|
@@ -67,7 +67,7 @@ module Rley # This module is used as a namespace
|
|
67
67
|
# builder.rule('A' => ['a', 'A', 'c']) # 'rule' is a synonym
|
68
68
|
# builder.rule('A' => %w[a A c]) # Use %w syntax for Array of String
|
69
69
|
# builder.rule 'A' => %w[a A c] # Call parentheses are optional
|
70
|
-
# @param aProductionRepr [Hash{String, Array<String>}]
|
70
|
+
# @param aProductionRepr [Hash{String, Array<String>}]
|
71
71
|
# A Hash-based representation of a production.
|
72
72
|
# @return [Production] The created Production instance
|
73
73
|
def add_production(aProductionRepr)
|
@@ -85,14 +85,14 @@ module Rley # This module is used as a namespace
|
|
85
85
|
new_prod = Production.new(lhs, rhs_members)
|
86
86
|
productions << new_prod
|
87
87
|
end
|
88
|
-
|
88
|
+
|
89
89
|
return productions.last
|
90
90
|
end
|
91
91
|
|
92
92
|
# Given the grammar symbols and productions added to the builder,
|
93
93
|
# build the resulting grammar (if not yet done).
|
94
|
-
# @return [Grammar] the created grammar object.
|
95
|
-
def grammar
|
94
|
+
# @return [Grammar] the created grammar object.
|
95
|
+
def grammar
|
96
96
|
unless @grammar
|
97
97
|
raise StandardError, 'No symbol found for grammar' if symbols.empty?
|
98
98
|
if productions.empty?
|
@@ -113,7 +113,7 @@ module Rley # This module is used as a namespace
|
|
113
113
|
unused = all_terminals.reject { |a_term| in_use.include?(a_term) }
|
114
114
|
unless unused.empty?
|
115
115
|
suffix = "#{unused.map(&:name).join(', ')}."
|
116
|
-
raise StandardError,
|
116
|
+
raise StandardError, "Useless terminal symbol(s): #{suffix}"
|
117
117
|
end
|
118
118
|
|
119
119
|
@grammar = Grammar.new(productions.dup)
|
@@ -166,7 +166,7 @@ module Rley # This module is used as a namespace
|
|
166
166
|
unless symbols.include? aSymbolName
|
167
167
|
symbols[aSymbolName] = NonTerminal.new(aSymbolName)
|
168
168
|
end
|
169
|
-
|
169
|
+
symbols[aSymbolName]
|
170
170
|
end
|
171
171
|
end # class
|
172
172
|
end # module
|
@@ -23,19 +23,19 @@ module Rley # This module is used as a namespace
|
|
23
23
|
|
24
24
|
# The String representation of the grammar symbol
|
25
25
|
# @return [String]
|
26
|
-
def to_s
|
27
|
-
|
26
|
+
def to_s
|
27
|
+
name.to_s
|
28
28
|
end
|
29
29
|
|
30
30
|
# @return [Boolean] true iff the symbol is a terminal
|
31
|
-
def terminal?
|
31
|
+
def terminal?
|
32
32
|
# Default implementation to override if necessary
|
33
|
-
|
33
|
+
false
|
34
34
|
end
|
35
35
|
|
36
36
|
# @return [Boolean] true iff the symbol is generative.
|
37
|
-
def generative?
|
38
|
-
|
37
|
+
def generative?
|
38
|
+
@generative
|
39
39
|
end
|
40
40
|
end # class
|
41
41
|
end # module
|
@@ -9,41 +9,35 @@ module Rley # This module is used as a namespace
|
|
9
9
|
class NonTerminal < GrmSymbol
|
10
10
|
# A non-terminal symbol is nullable if it can match an empty string.
|
11
11
|
attr_writer(:nullable)
|
12
|
-
|
12
|
+
|
13
13
|
# A non-terminal symbol is undefined if no production rule in the grammar
|
14
14
|
# has that non-terminal symbol in its left-hand side.
|
15
15
|
attr_writer(:undefined)
|
16
|
-
|
17
|
-
# A non-terminal symbol is unreachable if it cannot be reached (derived)
|
16
|
+
|
17
|
+
# A non-terminal symbol is unreachable if it cannot be reached (derived)
|
18
18
|
# from the start symbol.
|
19
19
|
attr_writer(:unreachable)
|
20
20
|
|
21
|
-
# Constructor.
|
22
|
-
# @param aName [String] The name of the grammar symbol.
|
23
|
-
def initialize(aName)
|
24
|
-
super(aName)
|
25
|
-
end
|
26
|
-
|
27
21
|
# @return [false/true] Return true if the symbol derives
|
28
22
|
# the empty string. As non-terminal symbol is nullable when it can
|
29
23
|
# can match to zero input token.
|
30
24
|
# The "nullability" of a non-terminal can practically be determined once
|
31
25
|
# all the production rules of the grammar are specified.
|
32
|
-
def nullable?
|
26
|
+
def nullable?
|
33
27
|
return @nullable
|
34
28
|
end
|
35
|
-
|
29
|
+
|
36
30
|
# @return [false/true] Return true if the symbol doesn't appear
|
37
31
|
# on the left-hand side of any production rule.
|
38
|
-
def undefined?
|
32
|
+
def undefined?
|
39
33
|
return @undefined
|
40
34
|
end
|
41
|
-
|
35
|
+
|
42
36
|
# @return [false/true] Return true if the symbol cannot be derived
|
43
37
|
# from the start symbol.
|
44
|
-
def unreachable?
|
38
|
+
def unreachable?
|
45
39
|
return @unreachable
|
46
|
-
end
|
40
|
+
end
|
47
41
|
end # class
|
48
42
|
end # module
|
49
43
|
end # module
|
@@ -46,33 +46,33 @@ module Rley # This module is used as a namespace
|
|
46
46
|
|
47
47
|
# Is the rhs empty?
|
48
48
|
# @return [Boolean] true if the rhs has no members.
|
49
|
-
def empty?
|
50
|
-
|
49
|
+
def empty?
|
50
|
+
rhs.empty?
|
51
51
|
end
|
52
52
|
|
53
53
|
# Return true iff the production is generative
|
54
|
-
def generative?
|
55
|
-
if @generative.nil?
|
56
|
-
end
|
54
|
+
def generative?
|
55
|
+
# if @generative.nil?
|
56
|
+
# end
|
57
57
|
|
58
|
-
|
58
|
+
@generative
|
59
59
|
end
|
60
60
|
|
61
61
|
# @return [Boolen] true iff the production is nullable
|
62
|
-
def nullable?
|
63
|
-
|
62
|
+
def nullable?
|
63
|
+
@nullable
|
64
64
|
end
|
65
65
|
|
66
66
|
# Returns a string containing a human-readable representation of the
|
67
67
|
# production.
|
68
68
|
# @return [String]
|
69
|
-
def inspect
|
69
|
+
def inspect
|
70
70
|
result = +"#<#{self.class.name}:#{object_id}"
|
71
71
|
result << " @name=\"#{name}\""
|
72
72
|
result << " @lhs=#{lhs.name}"
|
73
73
|
result << " @rhs=#{rhs.inspect}"
|
74
74
|
result << " @generative=#{@generative}>"
|
75
|
-
|
75
|
+
result
|
76
76
|
end
|
77
77
|
|
78
78
|
# A setter for the production name
|
@@ -12,9 +12,9 @@ module Rley # This module is used as a namespace
|
|
12
12
|
# @return [Array<GrmSymbol>] The sequence of symbols
|
13
13
|
attr_reader(:members)
|
14
14
|
|
15
|
-
# Create a sequence of grammar symbols (as in right-hand side of
|
15
|
+
# Create a sequence of grammar symbols (as in right-hand side of
|
16
16
|
# a production rule).
|
17
|
-
# @param theSymbols [Array<GrmSymbol>] An array of symbols.
|
17
|
+
# @param theSymbols [Array<GrmSymbol>] An array of symbols.
|
18
18
|
def initialize(theSymbols)
|
19
19
|
@members = theSymbols.dup
|
20
20
|
end
|
@@ -33,20 +33,18 @@ module Rley # This module is used as a namespace
|
|
33
33
|
raise StandardError, msg
|
34
34
|
end
|
35
35
|
|
36
|
-
|
36
|
+
result
|
37
37
|
end
|
38
|
-
|
39
|
-
# Returns a string containing a human-readable representation of the
|
38
|
+
|
39
|
+
# Returns a string containing a human-readable representation of the
|
40
40
|
# sequence of symbols.
|
41
41
|
# @return [String]
|
42
|
-
def inspect
|
42
|
+
def inspect
|
43
43
|
result = +"#<#{self.class.name}:#{object_id}"
|
44
44
|
symbol_names = members.map(&:name)
|
45
45
|
result << " @members=#{symbol_names}>"
|
46
|
-
|
46
|
+
result
|
47
47
|
end
|
48
|
-
|
49
|
-
|
50
48
|
end # class
|
51
49
|
end # module
|
52
50
|
end # module
|
data/lib/rley/syntax/terminal.rb
CHANGED
@@ -4,29 +4,28 @@ require_relative 'grm_symbol' # Load superclass
|
|
4
4
|
|
5
5
|
module Rley # This module is used as a namespace
|
6
6
|
module Syntax # This module is used as a namespace
|
7
|
-
# A terminal symbol represents a class of words in the language
|
7
|
+
# A terminal symbol represents a class of words in the language
|
8
8
|
# defined the grammar.
|
9
9
|
class Terminal < GrmSymbol
|
10
|
-
|
11
10
|
# Constructor.
|
12
11
|
# @param aName [String] The name of the grammar symbol.
|
13
12
|
def initialize(aName)
|
14
13
|
super(aName)
|
15
14
|
self.generative = true
|
16
15
|
end
|
17
|
-
|
16
|
+
|
18
17
|
# Return true iff the symbol is a terminal
|
19
18
|
def terminal?
|
20
19
|
return true
|
21
20
|
end
|
22
|
-
|
21
|
+
|
23
22
|
# @return [false] Return true if the symbol derives
|
24
23
|
# the empty string. As terminal symbol corresponds to a input token
|
25
24
|
# it is by definition non-nullable.
|
26
25
|
def nullable?
|
27
26
|
false
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def to_s
|
31
30
|
name
|
32
31
|
end
|
@@ -14,11 +14,11 @@ module Rley # This module is used as a namespace
|
|
14
14
|
super(aText) # Do we need to separate the text from the name?
|
15
15
|
@text = aText.dup
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
# The String representation of the verbatim symbol
|
19
19
|
# @return [String]
|
20
|
-
def to_s
|
21
|
-
|
20
|
+
def to_s
|
21
|
+
"'#{text}'"
|
22
22
|
end
|
23
23
|
end # class
|
24
24
|
end # module
|
@@ -3,21 +3,27 @@
|
|
3
3
|
require 'strscan'
|
4
4
|
require_relative '../rley/lexical/token'
|
5
5
|
|
6
|
+
# Simplistic tokenizer used mostly for testing purposes
|
6
7
|
class BaseTokenizer
|
8
|
+
# @return [StringScanner]
|
7
9
|
attr_reader(:scanner)
|
10
|
+
|
11
|
+
# @return [Integer] current line number
|
8
12
|
attr_reader(:lineno)
|
13
|
+
|
14
|
+
# @return [Integer] position of start of current line in source text
|
9
15
|
attr_reader(:line_start)
|
10
|
-
|
16
|
+
|
11
17
|
class ScanError < StandardError; end
|
12
18
|
|
13
|
-
# Constructor. Initialize a tokenizer
|
19
|
+
# Constructor. Initialize a tokenizer.
|
14
20
|
# @param source [String] Skeem text to tokenize.
|
15
21
|
def initialize(source)
|
16
22
|
@scanner = StringScanner.new('')
|
17
23
|
restart(source)
|
18
24
|
end
|
19
25
|
|
20
|
-
# @param source [String]
|
26
|
+
# @param source [String] input text to tokenize.
|
21
27
|
def restart(source)
|
22
28
|
@scanner.string = source
|
23
29
|
@lineno = 1
|
@@ -34,13 +40,13 @@ class BaseTokenizer
|
|
34
40
|
|
35
41
|
return tok_sequence
|
36
42
|
end
|
37
|
-
|
43
|
+
|
38
44
|
protected
|
39
|
-
|
45
|
+
|
40
46
|
# Patterns:
|
41
47
|
# Unambiguous single character
|
42
48
|
# Conditional single character:
|
43
|
-
# (e.g. '+' operator, '+' prefix for positive numbers)
|
49
|
+
# (e.g. '+' operator, '+' prefix for positive numbers)
|
44
50
|
def _next_token
|
45
51
|
skip_whitespaces
|
46
52
|
curr_ch = scanner.peek(1)
|
@@ -57,11 +63,11 @@ class BaseTokenizer
|
|
57
63
|
|
58
64
|
return token
|
59
65
|
end
|
60
|
-
|
66
|
+
|
61
67
|
def recognize_token
|
62
68
|
raise NotImplementedError
|
63
69
|
end
|
64
|
-
|
70
|
+
|
65
71
|
def build_token(aSymbolName, aLexeme, aFormat = :default)
|
66
72
|
begin
|
67
73
|
value = convert_to(aLexeme, aSymbolName, aFormat)
|
@@ -75,11 +81,11 @@ class BaseTokenizer
|
|
75
81
|
|
76
82
|
return token
|
77
83
|
end
|
78
|
-
|
84
|
+
|
79
85
|
def convert_to(aLexeme, _symbol_name, _format)
|
80
86
|
return aLexeme
|
81
87
|
end
|
82
|
-
|
88
|
+
|
83
89
|
def skip_whitespaces
|
84
90
|
pre_pos = scanner.pos
|
85
91
|
|
@@ -93,21 +99,16 @@ class BaseTokenizer
|
|
93
99
|
ws_found = true
|
94
100
|
next_line
|
95
101
|
end
|
96
|
-
|
97
|
-
# if next_ch == ';'
|
98
|
-
# cmt_found = true
|
99
|
-
# scanner.skip(/;[^\r\n]*(?:(?:\r\n)|\r|\n)?/)
|
100
|
-
# next_line
|
101
|
-
# end
|
102
|
+
|
102
103
|
break unless ws_found || cmt_found
|
103
104
|
end
|
104
105
|
|
105
106
|
curr_pos = scanner.pos
|
106
107
|
return if curr_pos == pre_pos
|
107
108
|
end
|
108
|
-
|
109
|
+
|
109
110
|
def next_line
|
110
111
|
@lineno += 1
|
111
112
|
@line_start = scanner.pos
|
112
|
-
end
|
113
|
+
end
|
113
114
|
end # class
|
@@ -125,7 +125,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
125
125
|
|
126
126
|
it 'should determine if it is a successor of another dotted item' do
|
127
127
|
expect(subject).not_to be_successor_of(subject)
|
128
|
-
|
128
|
+
|
129
129
|
# Case: different productions
|
130
130
|
instance = DottedItem.new(empty_prod, 0)
|
131
131
|
expect(subject).not_to be_successor_of(instance)
|
@@ -139,7 +139,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
139
139
|
instance2 = DottedItem.new(sample_prod, 2)
|
140
140
|
expect(instance).not_to be_successor_of(instance2)
|
141
141
|
expect(subject).not_to be_successor_of(instance2)
|
142
|
-
expect(instance2).to be_successor_of(subject)
|
142
|
+
expect(instance2).to be_successor_of(subject)
|
143
143
|
end
|
144
144
|
|
145
145
|
|
data/spec/rley/engine_spec.rb
CHANGED
@@ -19,11 +19,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
19
19
|
end
|
20
20
|
|
21
21
|
it 'could be created with block argument' do
|
22
|
-
expect do
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
22
|
+
expect do
|
23
|
+
Engine.new do |config|
|
24
|
+
config.parse_repr = :raw
|
25
|
+
end
|
26
|
+
end.not_to raise_error
|
27
27
|
end
|
28
28
|
|
29
29
|
it "shouldn't have a link to a grammar yet" do
|
@@ -45,13 +45,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
45
45
|
end
|
46
46
|
end # context
|
47
47
|
|
48
|
+
# rubocop: disable Lint/ConstantDefinitionInBlock
|
48
49
|
class ABCTokenizer
|
49
50
|
# Constructor
|
50
51
|
def initialize(someText)
|
51
52
|
@input = someText.dup
|
52
53
|
end
|
53
54
|
|
54
|
-
def each
|
55
|
+
def each
|
55
56
|
pos = Rley::Lexical::Position.new(1, 1) # Dummy position
|
56
57
|
lexemes = @input.scan(/\S/)
|
57
58
|
lexemes.each do |ch|
|
@@ -63,6 +64,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
63
64
|
end
|
64
65
|
end
|
65
66
|
end # class
|
67
|
+
# rubocop: enable Lint/ConstantDefinitionInBlock
|
66
68
|
|
67
69
|
# Utility method. Ensure that the engine
|
68
70
|
# has the defnition of a sample grammar
|
@@ -101,7 +103,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
101
103
|
sample_text = 'a a b c c'
|
102
104
|
ABCTokenizer.new(sample_text)
|
103
105
|
end
|
104
|
-
|
106
|
+
|
105
107
|
it 'should build a parse tree even for a nullable production' do
|
106
108
|
instance = Engine.new
|
107
109
|
instance.build_grammar do
|
@@ -112,12 +114,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
112
114
|
add_production 'B_opt' => 'b'
|
113
115
|
add_production 'B_opt' => []
|
114
116
|
add_production 'C_opt' => 'c'
|
115
|
-
add_production 'C_opt' => []
|
117
|
+
add_production 'C_opt' => []
|
116
118
|
end
|
117
119
|
input = ABCTokenizer.new('a')
|
118
120
|
raw_result = instance.parse(input)
|
119
121
|
expect { instance.to_ptree(raw_result) }.not_to raise_error
|
120
|
-
end
|
122
|
+
end
|
121
123
|
|
122
124
|
it 'should build default parse trees' do
|
123
125
|
raw_result = subject.parse(sample_tokenizer)
|
@@ -152,7 +154,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
152
154
|
sample_text = 'a a b c c'
|
153
155
|
ABCTokenizer.new(sample_text)
|
154
156
|
end
|
155
|
-
|
157
|
+
|
156
158
|
it 'should build a parse forest even for a nullable production' do
|
157
159
|
instance = Engine.new
|
158
160
|
instance.build_grammar do
|
@@ -163,19 +165,19 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
163
165
|
add_production 'B_opt' => 'b'
|
164
166
|
add_production 'B_opt' => []
|
165
167
|
add_production 'C_opt' => 'c'
|
166
|
-
add_production 'C_opt' => []
|
168
|
+
add_production 'C_opt' => []
|
167
169
|
end
|
168
170
|
input = ABCTokenizer.new('a')
|
169
171
|
raw_result = instance.parse(input)
|
170
172
|
expect { instance.to_pforest(raw_result) }.not_to raise_error
|
171
|
-
end
|
172
|
-
|
173
|
+
end
|
174
|
+
|
173
175
|
it 'should build parse forest' do
|
174
176
|
raw_result = subject.parse(sample_tokenizer)
|
175
177
|
pforest = subject.to_pforest(raw_result)
|
176
178
|
expect(pforest).to be_kind_of(SPPF::ParseForest)
|
177
|
-
end
|
178
|
-
|
179
|
+
end
|
180
|
+
|
179
181
|
it 'should provide a parse visitor' do
|
180
182
|
raw_result = subject.parse(sample_tokenizer)
|
181
183
|
ptree = subject.to_pforest(raw_result)
|