rley 0.8.06 → 0.8.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +23 -2
- data/CHANGELOG.md +21 -1
- data/LICENSE.txt +1 -1
- data/README.md +1 -1
- data/appveyor.yml +1 -3
- data/examples/NLP/benchmark_pico_en.rb +6 -6
- data/examples/NLP/engtagger.rb +6 -6
- data/examples/general/calc_iter1/calc_lexer.rb +1 -1
- data/examples/general/calc_iter2/calc_lexer.rb +1 -1
- data/examples/general/left.rb +1 -1
- data/examples/general/right.rb +1 -1
- data/examples/tokenizer/loxxy_raw_scanner.rex.rb +3 -0
- data/examples/tokenizer/loxxy_tokenizer.rb +2 -2
- data/examples/tokenizer/run_tokenizer.rb +1 -1
- data/examples/tokenizer/{tokens.yaml → tokens.yml} +0 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/engine.rb +2 -2
- data/lib/rley/interface.rb +3 -3
- data/lib/rley/lexical/token.rb +1 -1
- data/lib/rley/ptree/non_terminal_node.rb +1 -1
- data/lib/rley/rgn/all_notation_nodes.rb +5 -0
- data/lib/rley/{notation → rgn}/ast_builder.rb +19 -12
- data/lib/rley/{notation → rgn}/ast_node.rb +13 -12
- data/lib/rley/{notation → rgn}/ast_visitor.rb +10 -10
- data/lib/rley/rgn/composite_node.rb +28 -0
- data/lib/rley/{notation → rgn}/grammar.rb +1 -1
- data/lib/rley/{notation → rgn}/grammar_builder.rb +86 -124
- data/lib/rley/{notation → rgn}/parser.rb +7 -7
- data/lib/rley/rgn/repetition_node.rb +62 -0
- data/lib/rley/rgn/sequence_node.rb +30 -0
- data/lib/rley/{notation → rgn}/symbol_node.rb +15 -7
- data/lib/rley/{notation → rgn}/tokenizer.rb +71 -60
- data/lib/rley/syntax/grm_symbol.rb +0 -4
- data/lib/rley/syntax/non_terminal.rb +4 -0
- data/lib/rley/syntax/terminal.rb +10 -6
- data/spec/rley/parser/dangling_else_spec.rb +3 -3
- data/spec/rley/parser/gfg_earley_parser_spec.rb +48 -50
- data/spec/rley/{notation → rgn}/grammar_builder_spec.rb +58 -54
- data/spec/rley/{notation → rgn}/parser_spec.rb +36 -24
- data/spec/rley/rgn/repetition_node_spec.rb +56 -0
- data/spec/rley/rgn/sequence_node_spec.rb +48 -0
- data/spec/rley/rgn/symbol_node_spec.rb +33 -0
- data/spec/rley/{notation → rgn}/tokenizer_spec.rb +2 -2
- data/spec/rley/support/ambiguous_grammar_helper.rb +2 -2
- data/spec/rley/support/grammar_int_seq_helper.rb +2 -2
- metadata +40 -33
- data/lib/rley/notation/all_notation_nodes.rb +0 -4
- data/lib/rley/notation/grouping_node.rb +0 -23
- data/lib/rley/notation/sequence_node.rb +0 -35
@@ -0,0 +1,48 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
|
5
|
+
require_relative '../../../lib/rley/lexical/token'
|
6
|
+
require_relative '../../../lib/rley/rgn/symbol_node'
|
7
|
+
|
8
|
+
# Load the class under test
|
9
|
+
require_relative '../../../lib/rley/rgn/sequence_node'
|
10
|
+
|
11
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
12
|
+
module RGN # Open this namespace to avoid module qualifier prefixes
|
13
|
+
describe SequenceNode do
|
14
|
+
let(:name1) { 'LPAR' }
|
15
|
+
let(:name2) { 'arguments' }
|
16
|
+
let(:name3) { 'RPAR' }
|
17
|
+
let(:pos1) { Lexical::Position.new(3, 7) }
|
18
|
+
let(:pos2) { Lexical::Position.new(3, 9) }
|
19
|
+
let(:pos3) { Lexical::Position.new(3, 19) }
|
20
|
+
let(:child1) { SymbolNode.new(pos1, name1) }
|
21
|
+
let(:child2) { SymbolNode.new(pos2, name2) }
|
22
|
+
let(:child3) { SymbolNode.new(pos3, name3) }
|
23
|
+
|
24
|
+
# Default instantiation rule
|
25
|
+
subject { SequenceNode.new([child1, child2, child3]) }
|
26
|
+
|
27
|
+
context 'Initialization:' do
|
28
|
+
it 'should be created with an array of child nodes' do
|
29
|
+
children = [child1, child2, child3]
|
30
|
+
expect { SequenceNode.new(children) }.not_to raise_error
|
31
|
+
end
|
32
|
+
|
33
|
+
it 'should know its subnodes' do
|
34
|
+
expect(subject.subnodes.size).to eq(3)
|
35
|
+
expect(subject.subnodes).to eq([child1, child2, child3])
|
36
|
+
end
|
37
|
+
end # context
|
38
|
+
|
39
|
+
context 'Provided services:' do
|
40
|
+
it 'should know its name' do
|
41
|
+
expect(subject.name).to eq('seq_LPAR_arguments_RPAR')
|
42
|
+
end
|
43
|
+
end # context
|
44
|
+
end # describe
|
45
|
+
end # module
|
46
|
+
end # module
|
47
|
+
|
48
|
+
# End of file
|
@@ -0,0 +1,33 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative '../../spec_helper'
|
4
|
+
|
5
|
+
require_relative '../../../lib/rley/lexical/token'
|
6
|
+
|
7
|
+
# Load the class under test
|
8
|
+
require_relative '../../../lib/rley/rgn/symbol_node'
|
9
|
+
|
10
|
+
|
11
|
+
module Rley # Open this namespace to avoid module qualifier prefixes
|
12
|
+
module RGN # Open this namespace to avoid module qualifier prefixes
|
13
|
+
describe SymbolNode do
|
14
|
+
let(:a_name) { 'arguments' }
|
15
|
+
let(:a_pos) { Lexical::Position.new(3, 4) }
|
16
|
+
|
17
|
+
context 'Initialization:' do
|
18
|
+
# Default instantiation rule
|
19
|
+
subject { SymbolNode.new(a_pos, a_name) }
|
20
|
+
|
21
|
+
it 'should be created with a name and position' do
|
22
|
+
expect { SymbolNode.new(a_pos, a_name) }.not_to raise_error
|
23
|
+
end
|
24
|
+
|
25
|
+
it 'should know its name' do
|
26
|
+
expect(subject.name).to eq(a_name)
|
27
|
+
end
|
28
|
+
end # context
|
29
|
+
end # describe
|
30
|
+
end # module
|
31
|
+
end # module
|
32
|
+
|
33
|
+
# End of file
|
@@ -3,10 +3,10 @@
|
|
3
3
|
require_relative '../../spec_helper'
|
4
4
|
|
5
5
|
# Load the class under test
|
6
|
-
require_relative '../../../lib/rley/
|
6
|
+
require_relative '../../../lib/rley/rgn/tokenizer'
|
7
7
|
|
8
8
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
9
|
-
module
|
9
|
+
module RGN # Open this namespace to avoid module qualifier prefixes
|
10
10
|
describe Tokenizer do
|
11
11
|
# Utility method for comparing actual and expected token
|
12
12
|
# sequence. The final EOF is removed from the input sequence.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/
|
4
|
+
require_relative '../../../lib/rley/rgn/grammar_builder'
|
5
5
|
require_relative '../../../lib/rley/lexical/token'
|
6
6
|
|
7
7
|
|
@@ -10,7 +10,7 @@ module AmbiguousGrammarHelper
|
|
10
10
|
# expression grammar.
|
11
11
|
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
12
12
|
def grammar_builder
|
13
|
-
Rley::
|
13
|
+
Rley::RGN::GrammarBuilder.new do
|
14
14
|
add_terminals('+', 'id')
|
15
15
|
rule 'S' => 'E'
|
16
16
|
rule 'E' => 'E + E'
|
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'strscan'
|
4
4
|
|
5
5
|
# Load the builder class
|
6
|
-
require_relative '../../../lib/rley/
|
6
|
+
require_relative '../../../lib/rley/rgn/grammar_builder'
|
7
7
|
require_relative '../../../lib/rley/lexical/token'
|
8
8
|
|
9
9
|
|
@@ -11,7 +11,7 @@ module GrammarIntSeqHelper
|
|
11
11
|
# Factory method. Creates a builder for a grammar of sequence
|
12
12
|
# of positive integers.
|
13
13
|
def grammar_int_seq_builder
|
14
|
-
Rley::
|
14
|
+
Rley::RGN::GrammarBuilder.new do
|
15
15
|
add_terminals('comma', 'digit')
|
16
16
|
rule 'S' => 'sequence'
|
17
17
|
rule 'S' => ''
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rley
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dimitri Geshef
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-04-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -16,20 +16,20 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
19
|
+
version: '13.0'
|
20
20
|
- - ">="
|
21
21
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
22
|
+
version: 13.0.0
|
23
23
|
type: :development
|
24
24
|
prerelease: false
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - "~>"
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version: '
|
29
|
+
version: '13.0'
|
30
30
|
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version:
|
32
|
+
version: 13.0.0
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
34
|
name: rspec
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,7 +128,7 @@ files:
|
|
128
128
|
- examples/tokenizer/loxxy_raw_scanner.rex.rb
|
129
129
|
- examples/tokenizer/loxxy_tokenizer.rb
|
130
130
|
- examples/tokenizer/run_tokenizer.rb
|
131
|
-
- examples/tokenizer/tokens.
|
131
|
+
- examples/tokenizer/tokens.yml
|
132
132
|
- lib/rley.rb
|
133
133
|
- lib/rley/base/base_parser.rb
|
134
134
|
- lib/rley/base/dotted_item.rb
|
@@ -156,17 +156,6 @@ files:
|
|
156
156
|
- lib/rley/lexical/literal.rb
|
157
157
|
- lib/rley/lexical/token.rb
|
158
158
|
- lib/rley/lexical/token_range.rb
|
159
|
-
- lib/rley/notation/all_notation_nodes.rb
|
160
|
-
- lib/rley/notation/ast_builder.rb
|
161
|
-
- lib/rley/notation/ast_node.rb
|
162
|
-
- lib/rley/notation/ast_visitor.rb
|
163
|
-
- lib/rley/notation/grammar.rb
|
164
|
-
- lib/rley/notation/grammar_builder.rb
|
165
|
-
- lib/rley/notation/grouping_node.rb
|
166
|
-
- lib/rley/notation/parser.rb
|
167
|
-
- lib/rley/notation/sequence_node.rb
|
168
|
-
- lib/rley/notation/symbol_node.rb
|
169
|
-
- lib/rley/notation/tokenizer.rb
|
170
159
|
- lib/rley/parse_forest_visitor.rb
|
171
160
|
- lib/rley/parse_rep/ast_base_builder.rb
|
172
161
|
- lib/rley/parse_rep/cst_builder.rb
|
@@ -188,6 +177,18 @@ files:
|
|
188
177
|
- lib/rley/ptree/parse_tree.rb
|
189
178
|
- lib/rley/ptree/parse_tree_node.rb
|
190
179
|
- lib/rley/ptree/terminal_node.rb
|
180
|
+
- lib/rley/rgn/all_notation_nodes.rb
|
181
|
+
- lib/rley/rgn/ast_builder.rb
|
182
|
+
- lib/rley/rgn/ast_node.rb
|
183
|
+
- lib/rley/rgn/ast_visitor.rb
|
184
|
+
- lib/rley/rgn/composite_node.rb
|
185
|
+
- lib/rley/rgn/grammar.rb
|
186
|
+
- lib/rley/rgn/grammar_builder.rb
|
187
|
+
- lib/rley/rgn/parser.rb
|
188
|
+
- lib/rley/rgn/repetition_node.rb
|
189
|
+
- lib/rley/rgn/sequence_node.rb
|
190
|
+
- lib/rley/rgn/symbol_node.rb
|
191
|
+
- lib/rley/rgn/tokenizer.rb
|
191
192
|
- lib/rley/rley_error.rb
|
192
193
|
- lib/rley/sppf/alternative_node.rb
|
193
194
|
- lib/rley/sppf/composite_node.rb
|
@@ -228,9 +229,6 @@ files:
|
|
228
229
|
- spec/rley/lexical/literal_spec.rb
|
229
230
|
- spec/rley/lexical/token_range_spec.rb
|
230
231
|
- spec/rley/lexical/token_spec.rb
|
231
|
-
- spec/rley/notation/grammar_builder_spec.rb
|
232
|
-
- spec/rley/notation/parser_spec.rb
|
233
|
-
- spec/rley/notation/tokenizer_spec.rb
|
234
232
|
- spec/rley/parse_forest_visitor_spec.rb
|
235
233
|
- spec/rley/parse_rep/ambiguous_parse_spec.rb
|
236
234
|
- spec/rley/parse_rep/ast_builder_spec.rb
|
@@ -252,6 +250,12 @@ files:
|
|
252
250
|
- spec/rley/ptree/parse_tree_node_spec.rb
|
253
251
|
- spec/rley/ptree/parse_tree_spec.rb
|
254
252
|
- spec/rley/ptree/terminal_node_spec.rb
|
253
|
+
- spec/rley/rgn/grammar_builder_spec.rb
|
254
|
+
- spec/rley/rgn/parser_spec.rb
|
255
|
+
- spec/rley/rgn/repetition_node_spec.rb
|
256
|
+
- spec/rley/rgn/sequence_node_spec.rb
|
257
|
+
- spec/rley/rgn/symbol_node_spec.rb
|
258
|
+
- spec/rley/rgn/tokenizer_spec.rb
|
255
259
|
- spec/rley/sppf/alternative_node_spec.rb
|
256
260
|
- spec/rley/sppf/non_terminal_node_spec.rb
|
257
261
|
- spec/rley/sppf/token_node_spec.rb
|
@@ -299,7 +303,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
299
303
|
- !ruby/object:Gem::Version
|
300
304
|
version: '0'
|
301
305
|
requirements: []
|
302
|
-
rubygems_version: 3.
|
306
|
+
rubygems_version: 3.3.3
|
303
307
|
signing_key:
|
304
308
|
specification_version: 4
|
305
309
|
summary: Ruby implementation of the Earley's parsing algorithm
|
@@ -326,17 +330,6 @@ test_files:
|
|
326
330
|
- spec/rley/lexical/literal_spec.rb
|
327
331
|
- spec/rley/lexical/token_range_spec.rb
|
328
332
|
- spec/rley/lexical/token_spec.rb
|
329
|
-
- spec/rley/notation/grammar_builder_spec.rb
|
330
|
-
- spec/rley/notation/parser_spec.rb
|
331
|
-
- spec/rley/notation/tokenizer_spec.rb
|
332
|
-
- spec/rley/parser/dangling_else_spec.rb
|
333
|
-
- spec/rley/parser/error_reason_spec.rb
|
334
|
-
- spec/rley/parser/gfg_chart_spec.rb
|
335
|
-
- spec/rley/parser/gfg_earley_parser_spec.rb
|
336
|
-
- spec/rley/parser/gfg_parsing_spec.rb
|
337
|
-
- spec/rley/parser/parse_entry_set_spec.rb
|
338
|
-
- spec/rley/parser/parse_entry_spec.rb
|
339
|
-
- spec/rley/parser/parse_walker_factory_spec.rb
|
340
333
|
- spec/rley/parse_forest_visitor_spec.rb
|
341
334
|
- spec/rley/parse_rep/ambiguous_parse_spec.rb
|
342
335
|
- spec/rley/parse_rep/ast_builder_spec.rb
|
@@ -346,10 +339,24 @@ test_files:
|
|
346
339
|
- spec/rley/parse_rep/parse_forest_factory_spec.rb
|
347
340
|
- spec/rley/parse_rep/parse_tree_factory_spec.rb
|
348
341
|
- spec/rley/parse_tree_visitor_spec.rb
|
342
|
+
- spec/rley/parser/dangling_else_spec.rb
|
343
|
+
- spec/rley/parser/error_reason_spec.rb
|
344
|
+
- spec/rley/parser/gfg_chart_spec.rb
|
345
|
+
- spec/rley/parser/gfg_earley_parser_spec.rb
|
346
|
+
- spec/rley/parser/gfg_parsing_spec.rb
|
347
|
+
- spec/rley/parser/parse_entry_set_spec.rb
|
348
|
+
- spec/rley/parser/parse_entry_spec.rb
|
349
|
+
- spec/rley/parser/parse_walker_factory_spec.rb
|
349
350
|
- spec/rley/ptree/non_terminal_node_spec.rb
|
350
351
|
- spec/rley/ptree/parse_tree_node_spec.rb
|
351
352
|
- spec/rley/ptree/parse_tree_spec.rb
|
352
353
|
- spec/rley/ptree/terminal_node_spec.rb
|
354
|
+
- spec/rley/rgn/grammar_builder_spec.rb
|
355
|
+
- spec/rley/rgn/parser_spec.rb
|
356
|
+
- spec/rley/rgn/repetition_node_spec.rb
|
357
|
+
- spec/rley/rgn/sequence_node_spec.rb
|
358
|
+
- spec/rley/rgn/symbol_node_spec.rb
|
359
|
+
- spec/rley/rgn/tokenizer_spec.rb
|
353
360
|
- spec/rley/sppf/alternative_node_spec.rb
|
354
361
|
- spec/rley/sppf/non_terminal_node_spec.rb
|
355
362
|
- spec/rley/sppf/token_node_spec.rb
|
@@ -1,23 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative 'sequence_node'
|
4
|
-
|
5
|
-
module Rley
|
6
|
-
module Notation
|
7
|
-
# A syntax node representing an expression bracketed by parentheses.
|
8
|
-
class GroupingNode < SequenceNode
|
9
|
-
# @param aPosition [Rley::Lexical::Position] Start position.
|
10
|
-
# @param sequence [Array<ASTNode>] sequence of AST nodes
|
11
|
-
# @param theRepetition [Symbol] indicates how many times the symbol can be repeated
|
12
|
-
def initialize(aPosition, sequence, theRepetition = nil)
|
13
|
-
super(aPosition, sequence, theRepetition)
|
14
|
-
end
|
15
|
-
|
16
|
-
# Part of the 'visitee' role in Visitor design pattern.
|
17
|
-
# @param visitor [Notation::ASTVisitor] the visitor
|
18
|
-
def accept(visitor)
|
19
|
-
visitor.visit_grouping_node(self)
|
20
|
-
end
|
21
|
-
end # class
|
22
|
-
end # module
|
23
|
-
end # module
|
@@ -1,35 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative 'ast_node'
|
4
|
-
|
5
|
-
module Rley
|
6
|
-
module Notation
|
7
|
-
# A syntax node for a sequence of AST nodes
|
8
|
-
class SequenceNode < ASTNode
|
9
|
-
# @return [Array<ASTNode>]
|
10
|
-
attr_reader :subnodes
|
11
|
-
|
12
|
-
attr_accessor :constraints
|
13
|
-
|
14
|
-
# @param aPosition [Rley::Lexical::Position] Start position.
|
15
|
-
# @param sequence [Array<ASTNode>] sequence of AST nodes
|
16
|
-
# @param theRepetition [Symbol] indicates how many times the symbol can be repeated
|
17
|
-
def initialize(aPosition, sequence, theRepetition = nil)
|
18
|
-
super(aPosition)
|
19
|
-
@subnodes = sequence
|
20
|
-
self.repetition = theRepetition if theRepetition
|
21
|
-
@constraints = []
|
22
|
-
end
|
23
|
-
|
24
|
-
def size
|
25
|
-
subnodes.size
|
26
|
-
end
|
27
|
-
|
28
|
-
# Part of the 'visitee' role in Visitor design pattern.
|
29
|
-
# @param visitor [Notation::ASTVisitor] the visitor
|
30
|
-
def accept(visitor)
|
31
|
-
visitor.visit_sequence_node(self)
|
32
|
-
end
|
33
|
-
end # class
|
34
|
-
end # module
|
35
|
-
end # module
|