rley 0.7.08 → 0.8.03
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +29 -5
- data/CHANGELOG.md +28 -4
- data/README.md +4 -5
- data/examples/NLP/nano_eng/nano_en_demo.rb +7 -11
- data/examples/NLP/nano_eng/nano_grammar.rb +18 -18
- data/examples/data_formats/JSON/json_ast_builder.rb +9 -18
- data/examples/data_formats/JSON/json_demo.rb +1 -2
- data/examples/data_formats/JSON/json_grammar.rb +11 -11
- data/examples/general/calc_iter1/calc_grammar.rb +5 -4
- data/examples/general/calc_iter2/calc_grammar.rb +9 -9
- data/examples/general/left.rb +1 -1
- data/examples/general/right.rb +1 -1
- data/lib/rley/base/dotted_item.rb +5 -0
- data/lib/rley/base/grm_items_builder.rb +6 -0
- data/lib/rley/constants.rb +1 -1
- data/lib/rley/engine.rb +2 -2
- data/lib/rley/interface.rb +16 -0
- data/lib/rley/notation/all_notation_nodes.rb +4 -0
- data/lib/rley/notation/ast_builder.rb +185 -0
- data/lib/rley/notation/ast_node.rb +44 -0
- data/lib/rley/notation/ast_visitor.rb +115 -0
- data/lib/rley/notation/grammar.rb +49 -0
- data/lib/rley/notation/grammar_builder.rb +505 -0
- data/lib/rley/notation/grouping_node.rb +23 -0
- data/lib/rley/notation/parser.rb +56 -0
- data/lib/rley/notation/sequence_node.rb +35 -0
- data/lib/rley/notation/symbol_node.rb +29 -0
- data/lib/rley/notation/tokenizer.rb +180 -0
- data/lib/rley/parse_rep/ast_base_builder.rb +44 -0
- data/lib/rley/parser/gfg_chart.rb +101 -6
- data/lib/rley/parser/gfg_earley_parser.rb +1 -1
- data/lib/rley/parser/gfg_parsing.rb +5 -3
- data/lib/rley/parser/parse_entry_set.rb +1 -1
- data/lib/rley/syntax/{grammar_builder.rb → base_grammar_builder.rb} +53 -15
- data/lib/rley/syntax/grm_symbol.rb +1 -1
- data/lib/rley/syntax/match_closest.rb +43 -0
- data/lib/rley/syntax/production.rb +6 -0
- data/lib/rley.rb +1 -1
- data/spec/rley/engine_spec.rb +6 -6
- data/spec/rley/gfg/grm_flow_graph_spec.rb +2 -2
- data/spec/rley/notation/grammar_builder_spec.rb +302 -0
- data/spec/rley/notation/parser_spec.rb +183 -0
- data/spec/rley/notation/tokenizer_spec.rb +364 -0
- data/spec/rley/parse_rep/ast_builder_spec.rb +0 -1
- data/spec/rley/parse_rep/groucho_spec.rb +1 -1
- data/spec/rley/parse_rep/parse_forest_builder_spec.rb +1 -1
- data/spec/rley/parse_rep/parse_forest_factory_spec.rb +2 -2
- data/spec/rley/parse_rep/parse_tree_factory_spec.rb +1 -1
- data/spec/rley/parser/dangling_else_spec.rb +447 -0
- data/spec/rley/parser/gfg_earley_parser_spec.rb +118 -10
- data/spec/rley/parser/gfg_parsing_spec.rb +2 -1
- data/spec/rley/parser/parse_walker_factory_spec.rb +2 -2
- data/spec/rley/support/ambiguous_grammar_helper.rb +2 -2
- data/spec/rley/support/grammar_abc_helper.rb +2 -2
- data/spec/rley/support/grammar_ambig01_helper.rb +2 -2
- data/spec/rley/support/grammar_arr_int_helper.rb +2 -2
- data/spec/rley/support/grammar_b_expr_helper.rb +2 -2
- data/spec/rley/support/grammar_int_seq_helper.rb +51 -0
- data/spec/rley/support/grammar_l0_helper.rb +2 -2
- data/spec/rley/support/grammar_pb_helper.rb +2 -2
- data/spec/rley/support/grammar_sppf_helper.rb +2 -2
- data/spec/rley/syntax/{grammar_builder_spec.rb → base_grammar_builder_spec.rb} +29 -11
- data/spec/rley/syntax/match_closest_spec.rb +46 -0
- data/spec/rley/syntax/production_spec.rb +4 -0
- metadata +29 -14
- data/lib/rley/parser/parse_state.rb +0 -78
- data/lib/rley/parser/parse_state_tracker.rb +0 -59
- data/lib/rley/parser/state_set.rb +0 -100
- data/spec/rley/parser/parse_state_spec.rb +0 -125
- data/spec/rley/parser/parse_tracer_spec.rb +0 -200
- data/spec/rley/parser/state_set_spec.rb +0 -130
@@ -5,7 +5,7 @@ require 'stringio'
|
|
5
5
|
require_relative '../../../lib/rley/syntax/verbatim_symbol'
|
6
6
|
require_relative '../../../lib/rley/syntax/non_terminal'
|
7
7
|
require_relative '../../../lib/rley/syntax/production'
|
8
|
-
require_relative '../../../lib/rley/syntax/
|
8
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
9
9
|
require_relative '../../../lib/rley/lexical/token'
|
10
10
|
require_relative '../../../lib/rley/base/dotted_item'
|
11
11
|
require_relative '../../../lib/rley/parser/gfg_parsing'
|
@@ -13,6 +13,7 @@ require_relative '../../../lib/rley/parser/gfg_parsing'
|
|
13
13
|
# Load builders and lexers for sample grammars
|
14
14
|
require_relative '../support/grammar_abc_helper'
|
15
15
|
require_relative '../support/ambiguous_grammar_helper'
|
16
|
+
require_relative '../support/grammar_int_seq_helper'
|
16
17
|
require_relative '../support/grammar_pb_helper'
|
17
18
|
require_relative '../support/grammar_helper'
|
18
19
|
require_relative '../support/expectation_helper'
|
@@ -21,6 +22,7 @@ require_relative '../support/expectation_helper'
|
|
21
22
|
require_relative '../../../lib/rley/parser/gfg_earley_parser'
|
22
23
|
|
23
24
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
25
|
+
# rubocop: disable Metrics/BlockLength
|
24
26
|
module Parser # Open this namespace to avoid module qualifier prefixes
|
25
27
|
describe GFGEarleyParser do
|
26
28
|
include GrammarABCHelper # Mix-in module with builder for grammar abc
|
@@ -281,6 +283,91 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
281
283
|
compare_entry_texts(parse_result.chart[5], expected)
|
282
284
|
end
|
283
285
|
|
286
|
+
it 'should support Kleene plus ' do
|
287
|
+
extend(GrammarIntSeqHelper)
|
288
|
+
grammar = grammar_int_seq_builder.grammar
|
289
|
+
instance = GFGEarleyParser.new(grammar)
|
290
|
+
tokens = int_seq_tokenizer('6, 36, 216')
|
291
|
+
parse_result = nil
|
292
|
+
expect { parse_result = instance.parse(tokens) }.not_to raise_error
|
293
|
+
expect(parse_result.success?).to eq(true)
|
294
|
+
|
295
|
+
###################### S(0): . 6, 36, 216
|
296
|
+
# Expectation chart[0]:
|
297
|
+
expected = [
|
298
|
+
'.S | 0', # Initialization
|
299
|
+
'S => . sequence | 0', # start rule
|
300
|
+
'S => . | 0', # start rule
|
301
|
+
'.sequence | 0', # call rule
|
302
|
+
'S. | 0', # exit rule
|
303
|
+
'sequence => . sequence comma integer | 0', # start rule
|
304
|
+
'sequence => . integer | 0', # start rule
|
305
|
+
'.integer | 0', # call rule
|
306
|
+
'integer => . digit_plus | 0', # start rule
|
307
|
+
'.digit_plus | 0', # call rule
|
308
|
+
'digit_plus => . digit_plus digit | 0', # start rule (generated)
|
309
|
+
'digit_plus => . digit | 0' # start rule (generated)
|
310
|
+
]
|
311
|
+
compare_entry_texts(parse_result.chart[0], expected)
|
312
|
+
|
313
|
+
###################### S(1): 6 ., 36, 216
|
314
|
+
# Expectation chart[1]:
|
315
|
+
expected = [
|
316
|
+
'digit_plus => digit . | 0', # Scan
|
317
|
+
'digit_plus. | 0', # exit rule
|
318
|
+
'integer => digit_plus . | 0', # end rule
|
319
|
+
'digit_plus => digit_plus . digit | 0', # rule (generated)
|
320
|
+
'integer. | 0', # exit rule
|
321
|
+
'sequence => integer . | 0', # end rule
|
322
|
+
'sequence. | 0', # exit rule
|
323
|
+
'S => sequence . | 0', # end rule
|
324
|
+
'sequence => sequence . comma integer | 0', # rule
|
325
|
+
'S. | 0' # exit rule
|
326
|
+
]
|
327
|
+
compare_entry_texts(parse_result.chart[1], expected)
|
328
|
+
|
329
|
+
###################### S(2): 6 , . 36, 216
|
330
|
+
# Expectation chart[2]:
|
331
|
+
expected = [
|
332
|
+
'sequence => sequence comma . integer | 0', # Scan
|
333
|
+
'.integer | 2', # call rule
|
334
|
+
'integer => . digit_plus | 2', # start rule
|
335
|
+
'.digit_plus | 2', # call rule
|
336
|
+
'digit_plus => . digit_plus digit | 2', # start rule (generated)
|
337
|
+
'digit_plus => . digit | 2' # start rule (generated)
|
338
|
+
]
|
339
|
+
compare_entry_texts(parse_result.chart[2], expected)
|
340
|
+
|
341
|
+
###################### S(3): 6 , 3 . 6. , 216
|
342
|
+
# Expectation chart[3]:
|
343
|
+
expected = [
|
344
|
+
'digit_plus => digit . | 2', # Scan
|
345
|
+
'digit_plus. | 2', # exit rule
|
346
|
+
'integer => digit_plus . | 2', # end rule
|
347
|
+
'digit_plus => digit_plus . digit | 2', # rule (generated)
|
348
|
+
'integer. | 2', # exit rule
|
349
|
+
'sequence => sequence comma integer . | 0', # rule
|
350
|
+
'sequence. | 0', # exit rule
|
351
|
+
'S => sequence . | 0', # end rule
|
352
|
+
'sequence => sequence . comma integer | 0' # rule
|
353
|
+
]
|
354
|
+
compare_entry_texts(parse_result.chart[3], expected)
|
355
|
+
|
356
|
+
###################### S(4): 6 , 36 . . , 216
|
357
|
+
# Expectation chart[4]:
|
358
|
+
expected = [
|
359
|
+
'digit_plus => digit_plus digit . | 2', # Scan
|
360
|
+
'digit_plus. | 2', # exit rule
|
361
|
+
'integer => digit_plus . | 2', # end rule
|
362
|
+
'digit_plus => digit_plus . digit | 2', #
|
363
|
+
'integer. | 2', # exit rule
|
364
|
+
'sequence => sequence comma integer . | 0', # rule
|
365
|
+
'sequence. | 0', # exit rule
|
366
|
+
'S => sequence . | 0' # end rule
|
367
|
+
]
|
368
|
+
compare_entry_texts(parse_result.chart[4], expected)
|
369
|
+
end
|
370
|
+
|
284
371
|
it 'should parse a nullable grammar' do
|
285
372
|
# Simple but problematic grammar for the original Earley parser
|
286
373
|
# (based on example in D. Grune, C. Jacobs "Parsing Techniques" book)
|
@@ -288,7 +375,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
288
375
|
# A => ;
|
289
376
|
t_x = Syntax::VerbatimSymbol.new('x')
|
290
377
|
|
291
|
-
builder =
|
378
|
+
builder = Notation::GrammarBuilder.new do
|
292
379
|
add_terminals(t_x)
|
293
380
|
rule 'Ss' => 'A A x'
|
294
381
|
rule 'A' => []
|
@@ -334,11 +421,11 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
334
421
|
t_plus = Syntax::VerbatimSymbol.new('+')
|
335
422
|
t_star = Syntax::VerbatimSymbol.new('*')
|
336
423
|
|
337
|
-
builder = Syntax::
|
424
|
+
builder = Syntax::BaseGrammarBuilder.new do
|
338
425
|
add_terminals(t_int, t_plus, t_star)
|
339
426
|
rule 'P' => 'S'
|
340
|
-
rule 'S' =>
|
341
|
-
rule 'S' =>
|
427
|
+
rule 'S' => 'S + S'
|
428
|
+
rule 'S' => 'S * S'
|
342
429
|
rule 'S' => 'L'
|
343
430
|
rule 'L' => 'integer'
|
344
431
|
end
|
@@ -733,7 +820,7 @@ MSG
|
|
733
820
|
t_star = Syntax::VerbatimSymbol.new('*')
|
734
821
|
t_slash = Syntax::VerbatimSymbol.new('/')
|
735
822
|
|
736
|
-
builder = Syntax::
|
823
|
+
builder = Syntax::BaseGrammarBuilder.new do
|
737
824
|
add_terminals(t_a, t_star, t_slash)
|
738
825
|
rule 'Z' => 'E'
|
739
826
|
rule 'E' => %w[E Q F]
|
@@ -848,10 +935,11 @@ MSG
|
|
848
935
|
# S => ;
|
849
936
|
# This grammar requires a time that is quadratic in the number of
|
850
937
|
# input tokens
|
851
|
-
builder =
|
852
|
-
|
853
|
-
|
854
|
-
|
938
|
+
builder = Notation::GrammarBuilder.new do
|
939
|
+
add_terminals('a')
|
940
|
+
rule('S' => 'a S')
|
941
|
+
rule('S' => '')
|
942
|
+
end
|
855
943
|
grammar = builder.grammar
|
856
944
|
tokens = build_token_sequence(%w[a a a a], grammar)
|
857
945
|
|
@@ -931,8 +1019,28 @@ MSG
|
|
931
1019
|
]
|
932
1020
|
compare_entry_texts(parse_result.chart[4], expected)
|
933
1021
|
end
|
1022
|
+
|
1023
|
+
it 'should support modifier(s) in start rule' do
|
1024
|
+
# An implicit EOF marker is a special terminal
|
1025
|
+
# that denotes the end of input string but doesn't
|
1026
|
+
# appear explicitly as some character or text in the input.
|
1027
|
+
builder = Notation::GrammarBuilder.new do
|
1028
|
+
add_terminals('a', 'b', 'EOF')
|
1029
|
+
|
1030
|
+
rule('S' => 'a_or_b* EOF')
|
1031
|
+
rule('a_or_b' => 'a')
|
1032
|
+
rule('a_or_b' => 'b')
|
1033
|
+
end
|
1034
|
+
grammar = builder.grammar
|
1035
|
+
tokens = build_token_sequence(%w[EOF], grammar)
|
1036
|
+
tokens[0].instance_variable_set(:@lexeme, '')
|
1037
|
+
instance = GFGEarleyParser.new(grammar)
|
1038
|
+
parse_result = instance.parse(tokens)
|
1039
|
+
expect(parse_result.success?).to eq(true)
|
1040
|
+
end
|
934
1041
|
end # context
|
935
1042
|
end # describe
|
936
1043
|
end # module
|
1044
|
+
# rubocop: enable Metrics/BlockLength
|
937
1045
|
end # module
|
938
1046
|
# End of file
|
@@ -6,7 +6,7 @@ require 'stringio'
|
|
6
6
|
require_relative '../../../lib/rley/syntax/non_terminal'
|
7
7
|
require_relative '../../../lib/rley/syntax/verbatim_symbol'
|
8
8
|
require_relative '../../../lib/rley/syntax/production'
|
9
|
-
require_relative '../../../lib/rley/syntax/
|
9
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
10
10
|
require_relative '../../../lib/rley/base/dotted_item'
|
11
11
|
require_relative '../../../lib/rley/lexical/token'
|
12
12
|
require_relative '../../../lib/rley/parser/parse_tracer'
|
@@ -316,6 +316,7 @@ State[5]
|
|
316
316
|
S => S . + M | 0
|
317
317
|
P. | 0
|
318
318
|
REPR
|
319
|
+
expect(expectation).to eq(expectation)
|
319
320
|
end
|
320
321
|
end # context
|
321
322
|
|
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
require_relative '../../spec_helper'
|
4
4
|
|
5
|
-
require_relative '../../../lib/rley/syntax/
|
5
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
6
6
|
require_relative '../support/grammar_helper'
|
7
7
|
require_relative '../support/expectation_helper'
|
8
8
|
|
@@ -45,7 +45,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
45
45
|
# "SPPF=Style Parsing From Earley Recognizers" in
|
46
46
|
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
47
47
|
# contains a hidden left recursion and a cycle
|
48
|
-
builder = Syntax::
|
48
|
+
builder = Syntax::BaseGrammarBuilder.new do
|
49
49
|
add_terminals('a', 'b')
|
50
50
|
rule 'Phi' => 'S'
|
51
51
|
rule 'S' => %w[A T]
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/
|
4
|
+
require_relative '../../../lib/rley/notation/grammar_builder'
|
5
5
|
require_relative '../../../lib/rley/lexical/token'
|
6
6
|
|
7
7
|
|
@@ -10,7 +10,7 @@ module AmbiguousGrammarHelper
|
|
10
10
|
# expression grammar.
|
11
11
|
# (based on an example from Fisher and LeBlanc: "Crafting a Compiler")
|
12
12
|
def grammar_builder
|
13
|
-
Rley::
|
13
|
+
Rley::Notation::GrammarBuilder.new do
|
14
14
|
add_terminals('+', 'id')
|
15
15
|
rule 'S' => 'E'
|
16
16
|
rule 'E' => 'E + E'
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
|
6
6
|
module GrammarABCHelper
|
7
7
|
# Factory method. Creates a grammar builder for a simple grammar.
|
8
8
|
# (based on example in N. Wirth "Compiler Construction" book, p. 6)
|
9
9
|
def grammar_abc_builder
|
10
|
-
Rley::Syntax::
|
10
|
+
Rley::Syntax::BaseGrammarBuilder.new do
|
11
11
|
add_terminals('a', 'b', 'c')
|
12
12
|
rule 'S' => 'A'
|
13
13
|
rule 'A' => 'a A c'
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
require_relative '../../../lib/rley/lexical/token'
|
6
6
|
|
7
7
|
|
@@ -11,7 +11,7 @@ module GrammarAmbig01Helper
|
|
11
11
|
# Grammar 3: An ambiguous arithmetic expression language
|
12
12
|
# (based on example in article on Earley's algorithm in Wikipedia)
|
13
13
|
def grammar_ambig01_builder
|
14
|
-
Rley::Syntax::
|
14
|
+
Rley::Syntax::BaseGrammarBuilder.new do
|
15
15
|
add_terminals('integer', '+', '*')
|
16
16
|
rule 'P' => 'S'
|
17
17
|
rule 'S' => 'S + S'
|
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'strscan'
|
4
4
|
|
5
5
|
# Load the builder class
|
6
|
-
require_relative '../../../lib/rley/syntax/
|
6
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
7
7
|
require_relative '../../../lib/rley/lexical/token'
|
8
8
|
|
9
9
|
|
@@ -12,7 +12,7 @@ module GrammarArrIntHelper
|
|
12
12
|
# array of integers.
|
13
13
|
# (based on the article about Earley's algorithm in Wikipedia)
|
14
14
|
def grammar_arr_int_builder
|
15
|
-
Rley::Syntax::
|
15
|
+
Rley::Syntax::BaseGrammarBuilder.new do
|
16
16
|
add_terminals('[', ']', ',', 'integer')
|
17
17
|
rule 'P' => 'arr'
|
18
18
|
rule 'arr' => '[ sequence ]'
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
require_relative '../../../lib/rley/lexical/token'
|
6
6
|
|
7
7
|
|
@@ -10,7 +10,7 @@ module GrammarBExprHelper
|
|
10
10
|
# expression grammar.
|
11
11
|
# (based on the article about Earley's algorithm in Wikipedia)
|
12
12
|
def grammar_expr_builder
|
13
|
-
Rley::Syntax::
|
13
|
+
Rley::Syntax::BaseGrammarBuilder.new do
|
14
14
|
add_terminals('+', '*', 'integer')
|
15
15
|
rule 'P' => 'S'
|
16
16
|
rule 'S' => 'S + M'
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'strscan'
|
4
|
+
|
5
|
+
# Load the builder class
|
6
|
+
require_relative '../../../lib/rley/notation/grammar_builder'
|
7
|
+
require_relative '../../../lib/rley/lexical/token'
|
8
|
+
|
9
|
+
|
10
|
+
module GrammarIntSeqHelper
|
11
|
+
# Factory method. Creates a builder for a grammar of sequence
|
12
|
+
# of positive integers.
|
13
|
+
def grammar_int_seq_builder
|
14
|
+
Rley::Notation::GrammarBuilder.new do
|
15
|
+
add_terminals('comma', 'digit')
|
16
|
+
rule 'S' => 'sequence'
|
17
|
+
rule 'S' => ''
|
18
|
+
rule 'sequence' => 'sequence comma integer'
|
19
|
+
rule 'sequence' => 'integer'
|
20
|
+
rule 'integer' => 'digit+'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# Basic tokenizer for sequence positive integers
|
25
|
+
def int_seq_tokenizer(aText)
|
26
|
+
scanner = StringScanner.new(aText)
|
27
|
+
tokens = []
|
28
|
+
|
29
|
+
loop do
|
30
|
+
scanner.skip(/\s+/)
|
31
|
+
break if scanner.eos?
|
32
|
+
|
33
|
+
curr_pos = scanner.pos
|
34
|
+
|
35
|
+
if (lexeme = scanner.scan(/,/))
|
36
|
+
terminal = 'comma'
|
37
|
+
elsif (lexeme = scanner.scan(/\d/))
|
38
|
+
terminal = 'digit'
|
39
|
+
else
|
40
|
+
msg = "Unknown input text '#{scanner.scan(/.*/)}'"
|
41
|
+
raise StandardError, msg
|
42
|
+
end
|
43
|
+
|
44
|
+
pos = Rley::Lexical::Position.new(1, curr_pos + 1)
|
45
|
+
tokens << Rley::Lexical::Token.new(lexeme, terminal, pos)
|
46
|
+
end
|
47
|
+
|
48
|
+
return tokens
|
49
|
+
end
|
50
|
+
end # module
|
51
|
+
# End of file
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
require_relative '../../../lib/rley/lexical/token'
|
6
6
|
|
7
7
|
|
@@ -12,7 +12,7 @@ module GrammarL0Helper
|
|
12
12
|
# It defines the syntax of a sentence in a language with a
|
13
13
|
# very limited syntax and lexicon in the context of airline reservation.
|
14
14
|
def grammar_l0_builder
|
15
|
-
Rley::Syntax::
|
15
|
+
Rley::Syntax::BaseGrammarBuilder.new do
|
16
16
|
add_terminals('Noun', 'Verb', 'Pronoun', 'Proper-Noun')
|
17
17
|
add_terminals('Determiner', 'Preposition')
|
18
18
|
rule 'S' => 'NP VP'
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
require_relative '../../../lib/support/base_tokenizer'
|
6
6
|
require_relative '../../../lib/rley/lexical/token'
|
7
7
|
|
@@ -14,7 +14,7 @@ class GrammarPBHelper
|
|
14
14
|
# "A Graphical Model for Context-Free Grammar Parsing"
|
15
15
|
def grammar
|
16
16
|
@grammar ||= begin
|
17
|
-
builder = Rley::Syntax::
|
17
|
+
builder = Rley::Syntax::BaseGrammarBuilder.new do
|
18
18
|
add_terminals('int', '+', '(', ')')
|
19
19
|
rule 'S' => 'E'
|
20
20
|
rule 'E' => 'int'
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# Load the builder class
|
4
|
-
require_relative '../../../lib/rley/syntax/
|
4
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
5
5
|
|
6
6
|
|
7
7
|
module GrammarSPPFHelper
|
@@ -11,7 +11,7 @@ module GrammarSPPFHelper
|
|
11
11
|
# Notes in Theoretical Computer Science 203, (2008), pp. 53-67
|
12
12
|
# contains a hidden left recursion and a cycle
|
13
13
|
def grammar_sppf_builder
|
14
|
-
builder = Rley::Syntax::
|
14
|
+
builder = Rley::Syntax::BaseGrammarBuilder.new do
|
15
15
|
add_terminals('a', 'b')
|
16
16
|
rule 'Phi' => 'S'
|
17
17
|
rule 'S' => %w[A T]
|
@@ -3,14 +3,14 @@
|
|
3
3
|
require_relative '../../spec_helper'
|
4
4
|
|
5
5
|
# Load the class under test
|
6
|
-
require_relative '../../../lib/rley/syntax/
|
6
|
+
require_relative '../../../lib/rley/syntax/base_grammar_builder'
|
7
7
|
|
8
8
|
module Rley # Open this namespace to avoid module qualifier prefixes
|
9
9
|
module Syntax # Open this namespace to avoid module qualifier prefixes
|
10
|
-
describe
|
10
|
+
describe BaseGrammarBuilder do
|
11
11
|
context 'Initialization without argument:' do
|
12
12
|
it 'could be created without argument' do
|
13
|
-
expect {
|
13
|
+
expect { BaseGrammarBuilder.new }.not_to raise_error
|
14
14
|
end
|
15
15
|
|
16
16
|
it 'should have no grammar symbols at start' do
|
@@ -25,12 +25,12 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
25
25
|
context 'Initialization with argument:' do
|
26
26
|
it 'could be created with a block argument' do
|
27
27
|
expect do
|
28
|
-
|
28
|
+
BaseGrammarBuilder.new { nil }
|
29
29
|
end.not_to raise_error
|
30
30
|
end
|
31
31
|
|
32
32
|
it 'could have grammar symbols from block argument' do
|
33
|
-
instance =
|
33
|
+
instance = BaseGrammarBuilder.new do
|
34
34
|
add_terminals('a', 'b', 'c')
|
35
35
|
end
|
36
36
|
expect(instance.symbols.size).to eq(3)
|
@@ -80,7 +80,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
80
80
|
|
81
81
|
context 'Adding productions:' do
|
82
82
|
subject do
|
83
|
-
instance =
|
83
|
+
instance = BaseGrammarBuilder.new
|
84
84
|
instance.add_terminals('a', 'b', 'c')
|
85
85
|
instance
|
86
86
|
end
|
@@ -108,18 +108,36 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
108
108
|
expect_rhs = [subject['a'], subject['A'], subject['c']]
|
109
109
|
expect(new_prod.rhs.members).to eq(expect_rhs)
|
110
110
|
|
111
|
-
#
|
111
|
+
# BaseGrammarBuilder#rule is an alias of add_production
|
112
112
|
subject.rule('A' => ['b'])
|
113
113
|
expect(subject.productions.size).to eq(4)
|
114
114
|
new_prod = subject.productions.last
|
115
115
|
expect(new_prod.lhs).to eq(subject['A'])
|
116
116
|
expect(new_prod.rhs[0]).to eq(subject['b'])
|
117
117
|
end
|
118
|
+
|
119
|
+
it "should support Kleene's plus" do
|
120
|
+
instance = BaseGrammarBuilder.new
|
121
|
+
instance.add_terminals('plus', 'minus', 'digit')
|
122
|
+
|
123
|
+
instance.add_production('integer' => 'value')
|
124
|
+
instance.add_production('integer' => 'sign value')
|
125
|
+
instance.add_production('sign' => 'plus')
|
126
|
+
instance.add_production('sign' => 'minus')
|
127
|
+
expect(instance.productions.size).to eq(4)
|
128
|
+
instance.add_production('value' => 'digit+')
|
129
|
+
expect(instance.productions.size).to eq(7) # Two additional rules generated
|
130
|
+
prod_plus = instance.productions.select { |prod| prod.lhs.name == 'digit_plus' }
|
131
|
+
expect(prod_plus.size).to eq(2)
|
132
|
+
last_prod = instance.productions.last
|
133
|
+
expect(last_prod.lhs.name).to eq('value')
|
134
|
+
expect(last_prod.rhs.members[0].name).to eq('digit_plus')
|
135
|
+
end
|
118
136
|
end # context
|
119
137
|
|
120
138
|
context 'Building grammar:' do
|
121
139
|
subject do
|
122
|
-
instance =
|
140
|
+
instance = BaseGrammarBuilder.new do
|
123
141
|
add_terminals('a', 'b', 'c')
|
124
142
|
add_production('S' => ['A'])
|
125
143
|
add_production('A' => %w[a A c])
|
@@ -141,14 +159,14 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
141
159
|
end
|
142
160
|
|
143
161
|
it 'should complain in absence of symbols' do
|
144
|
-
instance =
|
162
|
+
instance = BaseGrammarBuilder.new
|
145
163
|
err = StandardError
|
146
164
|
msg = 'No symbol found for grammar'
|
147
165
|
expect { instance.grammar }.to raise_error(err, msg)
|
148
166
|
end
|
149
167
|
|
150
168
|
it 'should complain in absence of productions' do
|
151
|
-
instance =
|
169
|
+
instance = BaseGrammarBuilder.new
|
152
170
|
instance.add_terminals('a', 'b', 'c')
|
153
171
|
err = StandardError
|
154
172
|
msg = 'No production found for grammar'
|
@@ -183,7 +201,7 @@ module Rley # Open this namespace to avoid module qualifier prefixes
|
|
183
201
|
t_star = VerbatimSymbol.new('*')
|
184
202
|
t_slash = VerbatimSymbol.new('/')
|
185
203
|
|
186
|
-
builder =
|
204
|
+
builder = BaseGrammarBuilder.new
|
187
205
|
builder.add_terminals(t_a, t_star, t_slash)
|
188
206
|
builder.add_production('S' => 'E')
|
189
207
|
builder.add_production('E' => %w[E Q F])
|