rley 0.7.07 → 0.7.08
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +348 -54
- data/LICENSE.txt +1 -1
- data/README.md +3 -2
- data/examples/NLP/engtagger.rb +193 -190
- data/examples/NLP/nano_eng/nano_grammar.rb +5 -5
- data/examples/data_formats/JSON/cli_options.rb +1 -1
- data/examples/data_formats/JSON/json_ast_builder.rb +12 -9
- data/examples/data_formats/JSON/json_ast_nodes.rb +12 -21
- data/examples/data_formats/JSON/json_grammar.rb +2 -2
- data/examples/data_formats/JSON/json_lexer.rb +8 -8
- data/examples/data_formats/JSON/json_minifier.rb +1 -1
- data/examples/general/calc_iter1/calc_ast_builder.rb +13 -10
- data/examples/general/calc_iter1/calc_ast_nodes.rb +23 -37
- data/examples/general/calc_iter1/calc_grammar.rb +2 -2
- data/examples/general/calc_iter1/calc_lexer.rb +6 -4
- data/examples/general/calc_iter1/spec/calculator_spec.rb +5 -5
- data/examples/general/calc_iter2/calc_ast_builder.rb +5 -3
- data/examples/general/calc_iter2/calc_ast_nodes.rb +27 -43
- data/examples/general/calc_iter2/calc_grammar.rb +3 -3
- data/examples/general/calc_iter2/calc_lexer.rb +11 -10
- data/examples/general/calc_iter2/spec/calculator_spec.rb +26 -26
- data/examples/general/left.rb +2 -2
- data/examples/general/right.rb +2 -2
- data/lib/rley/base/dotted_item.rb +23 -31
- data/lib/rley/constants.rb +2 -2
- data/lib/rley/engine.rb +20 -23
- data/lib/rley/formatter/asciitree.rb +3 -3
- data/lib/rley/formatter/bracket_notation.rb +1 -8
- data/lib/rley/formatter/debug.rb +6 -6
- data/lib/rley/formatter/json.rb +2 -2
- data/lib/rley/gfg/call_edge.rb +1 -1
- data/lib/rley/gfg/edge.rb +5 -5
- data/lib/rley/gfg/end_vertex.rb +2 -6
- data/lib/rley/gfg/epsilon_edge.rb +1 -5
- data/lib/rley/gfg/grm_flow_graph.rb +27 -23
- data/lib/rley/gfg/item_vertex.rb +10 -10
- data/lib/rley/gfg/non_terminal_vertex.rb +4 -4
- data/lib/rley/gfg/scan_edge.rb +1 -1
- data/lib/rley/gfg/shortcut_edge.rb +2 -2
- data/lib/rley/gfg/start_vertex.rb +4 -8
- data/lib/rley/gfg/vertex.rb +43 -39
- data/lib/rley/lexical/token_range.rb +6 -6
- data/lib/rley/parse_forest_visitor.rb +5 -5
- data/lib/rley/parse_rep/ast_base_builder.rb +9 -11
- data/lib/rley/parse_rep/cst_builder.rb +5 -6
- data/lib/rley/parse_rep/parse_forest_builder.rb +20 -18
- data/lib/rley/parse_rep/parse_forest_factory.rb +3 -3
- data/lib/rley/parse_rep/parse_rep_creator.rb +11 -13
- data/lib/rley/parse_rep/parse_tree_builder.rb +4 -4
- data/lib/rley/parse_rep/parse_tree_factory.rb +27 -27
- data/lib/rley/parse_tree_visitor.rb +1 -1
- data/lib/rley/parser/error_reason.rb +4 -5
- data/lib/rley/parser/gfg_chart.rb +20 -22
- data/lib/rley/parser/gfg_parsing.rb +16 -30
- data/lib/rley/parser/parse_entry.rb +25 -31
- data/lib/rley/parser/parse_entry_set.rb +18 -15
- data/lib/rley/parser/parse_entry_tracker.rb +4 -4
- data/lib/rley/parser/parse_state.rb +16 -21
- data/lib/rley/parser/parse_state_tracker.rb +4 -4
- data/lib/rley/parser/parse_tracer.rb +13 -13
- data/lib/rley/parser/parse_walker_factory.rb +23 -28
- data/lib/rley/parser/state_set.rb +9 -10
- data/lib/rley/ptree/non_terminal_node.rb +7 -5
- data/lib/rley/ptree/parse_tree.rb +3 -3
- data/lib/rley/ptree/parse_tree_node.rb +5 -5
- data/lib/rley/ptree/terminal_node.rb +7 -7
- data/lib/rley/rley_error.rb +12 -12
- data/lib/rley/sppf/alternative_node.rb +6 -6
- data/lib/rley/sppf/composite_node.rb +7 -7
- data/lib/rley/sppf/epsilon_node.rb +3 -3
- data/lib/rley/sppf/leaf_node.rb +3 -3
- data/lib/rley/sppf/parse_forest.rb +16 -16
- data/lib/rley/sppf/sppf_node.rb +7 -8
- data/lib/rley/sppf/token_node.rb +3 -3
- data/lib/rley/syntax/grammar.rb +5 -5
- data/lib/rley/syntax/grammar_builder.rb +9 -9
- data/lib/rley/syntax/grm_symbol.rb +6 -6
- data/lib/rley/syntax/non_terminal.rb +9 -15
- data/lib/rley/syntax/production.rb +10 -10
- data/lib/rley/syntax/symbol_seq.rb +7 -9
- data/lib/rley/syntax/terminal.rb +4 -5
- data/lib/rley/syntax/verbatim_symbol.rb +3 -3
- data/lib/support/base_tokenizer.rb +19 -18
- data/spec/rley/base/dotted_item_spec.rb +2 -2
- data/spec/rley/engine_spec.rb +17 -15
- data/spec/rley/formatter/asciitree_spec.rb +7 -7
- data/spec/rley/formatter/bracket_notation_spec.rb +13 -13
- data/spec/rley/formatter/json_spec.rb +1 -1
- data/spec/rley/gfg/end_vertex_spec.rb +5 -5
- data/spec/rley/gfg/item_vertex_spec.rb +10 -10
- data/spec/rley/gfg/non_terminal_vertex_spec.rb +3 -3
- data/spec/rley/gfg/shortcut_edge_spec.rb +1 -1
- data/spec/rley/gfg/start_vertex_spec.rb +5 -5
- data/spec/rley/gfg/vertex_spec.rb +3 -3
- data/spec/rley/lexical/token_range_spec.rb +16 -16
- data/spec/rley/lexical/token_spec.rb +2 -2
- data/spec/rley/parse_forest_visitor_spec.rb +165 -163
- data/spec/rley/parse_rep/ambiguous_parse_spec.rb +44 -44
- data/spec/rley/parse_rep/ast_builder_spec.rb +6 -6
- data/spec/rley/parse_rep/cst_builder_spec.rb +5 -5
- data/spec/rley/parse_rep/groucho_spec.rb +21 -21
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +26 -26
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +6 -6
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +2 -2
- data/spec/rley/parse_tree_visitor_spec.rb +10 -8
- data/spec/rley/parser/error_reason_spec.rb +6 -6
- data/spec/rley/parser/gfg_earley_parser_spec.rb +4 -2
- data/spec/rley/parser/gfg_parsing_spec.rb +4 -8
- data/spec/rley/parser/parse_entry_spec.rb +19 -19
- data/spec/rley/parser/parse_state_spec.rb +5 -5
- data/spec/rley/parser/parse_walker_factory_spec.rb +1 -1
- data/spec/rley/parser/state_set_spec.rb +22 -22
- data/spec/rley/ptree/non_terminal_node_spec.rb +5 -3
- data/spec/rley/ptree/parse_tree_node_spec.rb +4 -4
- data/spec/rley/ptree/terminal_node_spec.rb +6 -6
- data/spec/rley/sppf/alternative_node_spec.rb +6 -6
- data/spec/rley/sppf/non_terminal_node_spec.rb +3 -3
- data/spec/rley/sppf/token_node_spec.rb +4 -4
- data/spec/rley/support/ambiguous_grammar_helper.rb +3 -4
- data/spec/rley/support/grammar_abc_helper.rb +2 -4
- data/spec/rley/support/grammar_ambig01_helper.rb +4 -5
- data/spec/rley/support/grammar_arr_int_helper.rb +4 -5
- data/spec/rley/support/grammar_b_expr_helper.rb +4 -5
- data/spec/rley/support/grammar_l0_helper.rb +10 -11
- data/spec/rley/support/grammar_pb_helper.rb +6 -5
- data/spec/rley/support/grammar_sppf_helper.rb +1 -1
- data/spec/rley/syntax/grammar_builder_spec.rb +5 -5
- data/spec/rley/syntax/grammar_spec.rb +6 -6
- data/spec/rley/syntax/grm_symbol_spec.rb +1 -1
- data/spec/rley/syntax/non_terminal_spec.rb +8 -8
- data/spec/rley/syntax/production_spec.rb +13 -13
- data/spec/rley/syntax/symbol_seq_spec.rb +2 -2
- data/spec/rley/syntax/terminal_spec.rb +5 -5
- data/spec/rley/syntax/verbatim_symbol_spec.rb +1 -1
- data/spec/spec_helper.rb +0 -12
- data/spec/support/base_tokenizer_spec.rb +7 -2
- metadata +21 -62
- data/.simplecov +0 -8
@@ -18,12 +18,12 @@ module Rley # This module is used as a namespace
|
|
18
18
|
# to the matching grammar symbol object.
|
19
19
|
attr_reader(:symbols)
|
20
20
|
|
21
|
-
# @return [Array<Production>] The list of production rules for
|
21
|
+
# @return [Array<Production>] The list of production rules for
|
22
22
|
# the grammar to build.
|
23
23
|
attr_reader(:productions)
|
24
24
|
|
25
25
|
# Creates a new grammar builder.
|
26
|
-
# @param aBlock [Proc] code block used to build the grammar.
|
26
|
+
# @param aBlock [Proc] code block used to build the grammar.
|
27
27
|
# @example Building a tiny English grammar
|
28
28
|
# builder = Rley::Syntax::GrammarBuilder.new do
|
29
29
|
# add_terminals('n', 'v', 'adj', 'det')
|
@@ -45,7 +45,7 @@ module Rley # This module is used as a namespace
|
|
45
45
|
# @param aSymbolName [String] the name of a grammar symbol.
|
46
46
|
# @return [GrmSymbol] the retrieved symbol object.
|
47
47
|
def [](aSymbolName)
|
48
|
-
|
48
|
+
symbols[aSymbolName]
|
49
49
|
end
|
50
50
|
|
51
51
|
# Add the given terminal symbols to the grammar of the language
|
@@ -67,7 +67,7 @@ module Rley # This module is used as a namespace
|
|
67
67
|
# builder.rule('A' => ['a', 'A', 'c']) # 'rule' is a synonym
|
68
68
|
# builder.rule('A' => %w[a A c]) # Use %w syntax for Array of String
|
69
69
|
# builder.rule 'A' => %w[a A c] # Call parentheses are optional
|
70
|
-
# @param aProductionRepr [Hash{String, Array<String>}]
|
70
|
+
# @param aProductionRepr [Hash{String, Array<String>}]
|
71
71
|
# A Hash-based representation of a production.
|
72
72
|
# @return [Production] The created Production instance
|
73
73
|
def add_production(aProductionRepr)
|
@@ -85,14 +85,14 @@ module Rley # This module is used as a namespace
|
|
85
85
|
new_prod = Production.new(lhs, rhs_members)
|
86
86
|
productions << new_prod
|
87
87
|
end
|
88
|
-
|
88
|
+
|
89
89
|
return productions.last
|
90
90
|
end
|
91
91
|
|
92
92
|
# Given the grammar symbols and productions added to the builder,
|
93
93
|
# build the resulting grammar (if not yet done).
|
94
|
-
# @return [Grammar] the created grammar object.
|
95
|
-
def grammar
|
94
|
+
# @return [Grammar] the created grammar object.
|
95
|
+
def grammar
|
96
96
|
unless @grammar
|
97
97
|
raise StandardError, 'No symbol found for grammar' if symbols.empty?
|
98
98
|
if productions.empty?
|
@@ -113,7 +113,7 @@ module Rley # This module is used as a namespace
|
|
113
113
|
unused = all_terminals.reject { |a_term| in_use.include?(a_term) }
|
114
114
|
unless unused.empty?
|
115
115
|
suffix = "#{unused.map(&:name).join(', ')}."
|
116
|
-
raise StandardError,
|
116
|
+
raise StandardError, "Useless terminal symbol(s): #{suffix}"
|
117
117
|
end
|
118
118
|
|
119
119
|
@grammar = Grammar.new(productions.dup)
|
@@ -166,7 +166,7 @@ module Rley # This module is used as a namespace
|
|
166
166
|
unless symbols.include? aSymbolName
|
167
167
|
symbols[aSymbolName] = NonTerminal.new(aSymbolName)
|
168
168
|
end
|
169
|
-
|
169
|
+
symbols[aSymbolName]
|
170
170
|
end
|
171
171
|
end # class
|
172
172
|
end # module
|
@@ -23,19 +23,19 @@ module Rley # This module is used as a namespace
|
|
23
23
|
|
24
24
|
# The String representation of the grammar symbol
|
25
25
|
# @return [String]
|
26
|
-
def to_s
|
27
|
-
|
26
|
+
def to_s
|
27
|
+
name.to_s
|
28
28
|
end
|
29
29
|
|
30
30
|
# @return [Boolean] true iff the symbol is a terminal
|
31
|
-
def terminal?
|
31
|
+
def terminal?
|
32
32
|
# Default implementation to override if necessary
|
33
|
-
|
33
|
+
false
|
34
34
|
end
|
35
35
|
|
36
36
|
# @return [Boolean] true iff the symbol is generative.
|
37
|
-
def generative?
|
38
|
-
|
37
|
+
def generative?
|
38
|
+
@generative
|
39
39
|
end
|
40
40
|
end # class
|
41
41
|
end # module
|
@@ -9,41 +9,35 @@ module Rley # This module is used as a namespace
|
|
9
9
|
class NonTerminal < GrmSymbol
|
10
10
|
# A non-terminal symbol is nullable if it can match an empty string.
|
11
11
|
attr_writer(:nullable)
|
12
|
-
|
12
|
+
|
13
13
|
# A non-terminal symbol is undefined if no production rule in the grammar
|
14
14
|
# has that non-terminal symbol in its left-hand side.
|
15
15
|
attr_writer(:undefined)
|
16
|
-
|
17
|
-
# A non-terminal symbol is unreachable if it cannot be reached (derived)
|
16
|
+
|
17
|
+
# A non-terminal symbol is unreachable if it cannot be reached (derived)
|
18
18
|
# from the start symbol.
|
19
19
|
attr_writer(:unreachable)
|
20
20
|
|
21
|
-
# Constructor.
|
22
|
-
# @param aName [String] The name of the grammar symbol.
|
23
|
-
def initialize(aName)
|
24
|
-
super(aName)
|
25
|
-
end
|
26
|
-
|
27
21
|
# @return [false/true] Return true if the symbol derives
|
28
22
|
# the empty string. As non-terminal symbol is nullable when it can
|
29
23
|
# can match to zero input token.
|
30
24
|
# The "nullability" of a non-terminal can practically be determined once
|
31
25
|
# all the production rules of the grammar are specified.
|
32
|
-
def nullable?
|
26
|
+
def nullable?
|
33
27
|
return @nullable
|
34
28
|
end
|
35
|
-
|
29
|
+
|
36
30
|
# @return [false/true] Return true if the symbol doesn't appear
|
37
31
|
# on the left-hand side of any production rule.
|
38
|
-
def undefined?
|
32
|
+
def undefined?
|
39
33
|
return @undefined
|
40
34
|
end
|
41
|
-
|
35
|
+
|
42
36
|
# @return [false/true] Return true if the symbol cannot be derived
|
43
37
|
# from the start symbol.
|
44
|
-
def unreachable?
|
38
|
+
def unreachable?
|
45
39
|
return @unreachable
|
46
|
-
end
|
40
|
+
end
|
47
41
|
end # class
|
48
42
|
end # module
|
49
43
|
end # module
|
@@ -46,33 +46,33 @@ module Rley # This module is used as a namespace
|
|
46
46
|
|
47
47
|
# Is the rhs empty?
|
48
48
|
# @return [Boolean] true if the rhs has no members.
|
49
|
-
def empty?
|
50
|
-
|
49
|
+
def empty?
|
50
|
+
rhs.empty?
|
51
51
|
end
|
52
52
|
|
53
53
|
# Return true iff the production is generative
|
54
|
-
def generative?
|
55
|
-
if @generative.nil?
|
56
|
-
end
|
54
|
+
def generative?
|
55
|
+
# if @generative.nil?
|
56
|
+
# end
|
57
57
|
|
58
|
-
|
58
|
+
@generative
|
59
59
|
end
|
60
60
|
|
61
61
|
# @return [Boolen] true iff the production is nullable
|
62
|
-
def nullable?
|
63
|
-
|
62
|
+
def nullable?
|
63
|
+
@nullable
|
64
64
|
end
|
65
65
|
|
66
66
|
# Returns a string containing a human-readable representation of the
|
67
67
|
# production.
|
68
68
|
# @return [String]
|
69
|
-
def inspect
|
69
|
+
def inspect
|
70
70
|
result = +"#<#{self.class.name}:#{object_id}"
|
71
71
|
result << " @name=\"#{name}\""
|
72
72
|
result << " @lhs=#{lhs.name}"
|
73
73
|
result << " @rhs=#{rhs.inspect}"
|
74
74
|
result << " @generative=#{@generative}>"
|
75
|
-
|
75
|
+
result
|
76
76
|
end
|
77
77
|
|
78
78
|
# A setter for the production name
|
@@ -12,9 +12,9 @@ module Rley # This module is used as a namespace
|
|
12
12
|
# @return [Array<GrmSymbol>] The sequence of symbols
|
13
13
|
attr_reader(:members)
|
14
14
|
|
15
|
-
# Create a sequence of grammar symbols (as in right-hand side of
|
15
|
+
# Create a sequence of grammar symbols (as in right-hand side of
|
16
16
|
# a production rule).
|
17
|
-
# @param theSymbols [Array<GrmSymbol>] An array of symbols.
|
17
|
+
# @param theSymbols [Array<GrmSymbol>] An array of symbols.
|
18
18
|
def initialize(theSymbols)
|
19
19
|
@members = theSymbols.dup
|
20
20
|
end
|
@@ -33,20 +33,18 @@ module Rley # This module is used as a namespace
|
|
33
33
|
raise StandardError, msg
|
34
34
|
end
|
35
35
|
|
36
|
-
|
36
|
+
result
|
37
37
|
end
|
38
|
-
|
39
|
-
# Returns a string containing a human-readable representation of the
|
38
|
+
|
39
|
+
# Returns a string containing a human-readable representation of the
|
40
40
|
# sequence of symbols.
|
41
41
|
# @return [String]
|
42
|
-
def inspect
|
42
|
+
def inspect
|
43
43
|
result = +"#<#{self.class.name}:#{object_id}"
|
44
44
|
symbol_names = members.map(&:name)
|
45
45
|
result << " @members=#{symbol_names}>"
|
46
|
-
|
46
|
+
result
|
47
47
|
end
|
48
|
-
|
49
|
-
|
50
48
|
end # class
|
51
49
|
end # module
|
52
50
|
end # module
|
data/lib/rley/syntax/terminal.rb
CHANGED
@@ -4,29 +4,28 @@ require_relative 'grm_symbol' # Load superclass
|
|
4
4
|
|
5
5
|
module Rley # This module is used as a namespace
|
6
6
|
module Syntax # This module is used as a namespace
|
7
|
-
# A terminal symbol represents a class of words in the language
|
7
|
+
# A terminal symbol represents a class of words in the language
|
8
8
|
# defined the grammar.
|
9
9
|
class Terminal < GrmSymbol
|
10
|
-
|
11
10
|
# Constructor.
|
12
11
|
# @param aName [String] The name of the grammar symbol.
|
13
12
|
def initialize(aName)
|
14
13
|
super(aName)
|
15
14
|
self.generative = true
|
16
15
|
end
|
17
|
-
|
16
|
+
|
18
17
|
# Return true iff the symbol is a terminal
|
19
18
|
def terminal?
|
20
19
|
return true
|
21
20
|
end
|
22
|
-
|
21
|
+
|
23
22
|
# @return [false] Return true if the symbol derives
|
24
23
|
# the empty string. As terminal symbol corresponds to a input token
|
25
24
|
# it is by definition non-nullable.
|
26
25
|
def nullable?
|
27
26
|
false
|
28
27
|
end
|
29
|
-
|
28
|
+
|
30
29
|
def to_s
|
31
30
|
name
|
32
31
|
end
|
@@ -14,11 +14,11 @@ module Rley # This module is used as a namespace
|
|
14
14
|
super(aText) # Do we need to separate the text from the name?
|
15
15
|
@text = aText.dup
|
16
16
|
end
|
17
|
-
|
17
|
+
|
18
18
|
# The String representation of the verbatim symbol
|
19
19
|
# @return [String]
|
20
|
-
def to_s
|
21
|
-
|
20
|
+
def to_s
|
21
|
+
"'#{text}'"
|
22
22
|
end
|
23
23
|
end # class
|
24
24
|
end # module
|
@@ -3,21 +3,27 @@
|
|
3
3
|
require 'strscan'
|
4
4
|
require_relative '../rley/lexical/token'
|
5
5
|
|
6
|
+
# Simplistic tokenizer used mostly for testing purposes
|
6
7
|
class BaseTokenizer
|
8
|
+
# @return [StringScanner]
|
7
9
|
attr_reader(:scanner)
|
10
|
+
|
11
|
+
# @return [Integer] current line number
|
8
12
|
attr_reader(:lineno)
|
13
|
+
|
14
|
+
# @return [Integer] position of start of current line in source text
|
9
15
|
attr_reader(:line_start)
|
10
|
-
|
16
|
+
|
11
17
|
class ScanError < StandardError; end
|
12
18
|
|
13
|
-
# Constructor. Initialize a tokenizer
|
19
|
+
# Constructor. Initialize a tokenizer.
|
14
20
|
# @param source [String] Skeem text to tokenize.
|
15
21
|
def initialize(source)
|
16
22
|
@scanner = StringScanner.new('')
|
17
23
|
restart(source)
|
18
24
|
end
|
19
25
|
|
20
|
-
# @param source [String]
|
26
|
+
# @param source [String] input text to tokenize.
|
21
27
|
def restart(source)
|
22
28
|
@scanner.string = source
|
23
29
|
@lineno = 1
|
@@ -34,13 +40,13 @@ class BaseTokenizer
|
|
34
40
|
|
35
41
|
return tok_sequence
|
36
42
|
end
|
37
|
-
|
43
|
+
|
38
44
|
protected
|
39
|
-
|
45
|
+
|
40
46
|
# Patterns:
|
41
47
|
# Unambiguous single character
|
42
48
|
# Conditional single character:
|
43
|
-
# (e.g. '+' operator, '+' prefix for positive numbers)
|
49
|
+
# (e.g. '+' operator, '+' prefix for positive numbers)
|
44
50
|
def _next_token
|
45
51
|
skip_whitespaces
|
46
52
|
curr_ch = scanner.peek(1)
|
@@ -57,11 +63,11 @@ class BaseTokenizer
|
|
57
63
|
|
58
64
|
return token
|
59
65
|
end
|
60
|
-
|
66
|
+
|
61
67
|
def recognize_token
|
62
68
|
raise NotImplementedError
|
63
69
|
end
|
64
|
-
|
70
|
+
|
65
71
|
def build_token(aSymbolName, aLexeme, aFormat = :default)
|
66
72
|
begin
|
67
73
|
value = convert_to(aLexeme, aSymbolName, aFormat)
|
@@ -75,11 +81,11 @@ class BaseTokenizer
|
|
75
81
|
|
76
82
|
return token
|
77
83
|
end
|
78
|
-
|
84
|
+
|
79
85
|
def convert_to(aLexeme, _symbol_name, _format)
|
80
86
|
return aLexeme
|
81
87
|
end
|
82
|
-
|
88
|
+
|
83
89
|
def skip_whitespaces
|
84
90
|
pre_pos = scanner.pos
|
85
91
|
|
@@ -93,21 +99,16 @@ class BaseTokenizer
|
|
93
99
|
ws_found = true
|
94
100
|
next_line
|
95
101
|
end
|
96
|
-
|
97
|
-
# if next_ch == ';'
|
98
|
-
# cmt_found = true
|
99
|
-
# scanner.skip(/;[^\r\n]*(?:(?:\r\n)|\r|\n)?/)
|
100
|
-
# next_line
|
101
|
-
# end
|
102
|
+
|
102
103
|
break unless ws_found || cmt_found
|
103
104
|
end
|
104
105
|
|
105
106
|
curr_pos = scanner.pos
|
106
107
|
return if curr_pos == pre_pos
|
107
108
|
end
|
108
|
-
|
109
|
+
|
109
110
|
def next_line
|
110
111
|
@lineno += 1
|
111
112
|
@line_start = scanner.pos
|
112
|
-
end
|
113
|
+
end
|
113
114
|
end # class
|
@@ -125,7 +125,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
125
125
|
|
126
126
|
it 'should determine if it is a successor of another dotted item' do
|
127
127
|
expect(subject).not_to be_successor_of(subject)
|
128
|
-
|
128
|
+
|
129
129
|
# Case: different productions
|
130
130
|
instance = DottedItem.new(empty_prod, 0)
|
131
131
|
expect(subject).not_to be_successor_of(instance)
|
@@ -139,7 +139,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
139
139
|
instance2 = DottedItem.new(sample_prod, 2)
|
140
140
|
expect(instance).not_to be_successor_of(instance2)
|
141
141
|
expect(subject).not_to be_successor_of(instance2)
|
142
|
-
expect(instance2).to be_successor_of(subject)
|
142
|
+
expect(instance2).to be_successor_of(subject)
|
143
143
|
end
|
144
144
|
|
145
145
|
|
data/spec/rley/engine_spec.rb
CHANGED
@@ -19,11 +19,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
19
19
|
end
|
20
20
|
|
21
21
|
it 'could be created with block argument' do
|
22
|
-
expect do
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
22
|
+
expect do
|
23
|
+
Engine.new do |config|
|
24
|
+
config.parse_repr = :raw
|
25
|
+
end
|
26
|
+
end.not_to raise_error
|
27
27
|
end
|
28
28
|
|
29
29
|
it "shouldn't have a link to a grammar yet" do
|
@@ -45,13 +45,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
45
45
|
end
|
46
46
|
end # context
|
47
47
|
|
48
|
+
# rubocop: disable Lint/ConstantDefinitionInBlock
|
48
49
|
class ABCTokenizer
|
49
50
|
# Constructor
|
50
51
|
def initialize(someText)
|
51
52
|
@input = someText.dup
|
52
53
|
end
|
53
54
|
|
54
|
-
def each
|
55
|
+
def each
|
55
56
|
pos = Rley::Lexical::Position.new(1, 1) # Dummy position
|
56
57
|
lexemes = @input.scan(/\S/)
|
57
58
|
lexemes.each do |ch|
|
@@ -63,6 +64,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
63
64
|
end
|
64
65
|
end
|
65
66
|
end # class
|
67
|
+
# rubocop: enable Lint/ConstantDefinitionInBlock
|
66
68
|
|
67
69
|
# Utility method. Ensure that the engine
|
68
70
|
# has the defnition of a sample grammar
|
@@ -101,7 +103,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
101
103
|
sample_text = 'a a b c c'
|
102
104
|
ABCTokenizer.new(sample_text)
|
103
105
|
end
|
104
|
-
|
106
|
+
|
105
107
|
it 'should build a parse tree even for a nullable production' do
|
106
108
|
instance = Engine.new
|
107
109
|
instance.build_grammar do
|
@@ -112,12 +114,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
112
114
|
add_production 'B_opt' => 'b'
|
113
115
|
add_production 'B_opt' => []
|
114
116
|
add_production 'C_opt' => 'c'
|
115
|
-
add_production 'C_opt' => []
|
117
|
+
add_production 'C_opt' => []
|
116
118
|
end
|
117
119
|
input = ABCTokenizer.new('a')
|
118
120
|
raw_result = instance.parse(input)
|
119
121
|
expect { instance.to_ptree(raw_result) }.not_to raise_error
|
120
|
-
end
|
122
|
+
end
|
121
123
|
|
122
124
|
it 'should build default parse trees' do
|
123
125
|
raw_result = subject.parse(sample_tokenizer)
|
@@ -152,7 +154,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
152
154
|
sample_text = 'a a b c c'
|
153
155
|
ABCTokenizer.new(sample_text)
|
154
156
|
end
|
155
|
-
|
157
|
+
|
156
158
|
it 'should build a parse forest even for a nullable production' do
|
157
159
|
instance = Engine.new
|
158
160
|
instance.build_grammar do
|
@@ -163,19 +165,19 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
163
165
|
add_production 'B_opt' => 'b'
|
164
166
|
add_production 'B_opt' => []
|
165
167
|
add_production 'C_opt' => 'c'
|
166
|
-
add_production 'C_opt' => []
|
168
|
+
add_production 'C_opt' => []
|
167
169
|
end
|
168
170
|
input = ABCTokenizer.new('a')
|
169
171
|
raw_result = instance.parse(input)
|
170
172
|
expect { instance.to_pforest(raw_result) }.not_to raise_error
|
171
|
-
end
|
172
|
-
|
173
|
+
end
|
174
|
+
|
173
175
|
it 'should build parse forest' do
|
174
176
|
raw_result = subject.parse(sample_tokenizer)
|
175
177
|
pforest = subject.to_pforest(raw_result)
|
176
178
|
expect(pforest).to be_kind_of(SPPF::ParseForest)
|
177
|
-
end
|
178
|
-
|
179
|
+
end
|
180
|
+
|
179
181
|
it 'should provide a parse visitor' do
|
180
182
|
raw_result = subject.parse(sample_tokenizer)
|
181
183
|
ptree = subject.to_pforest(raw_result)
|