rattler 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.rdoc +2 -0
- data/bin/rtlr +8 -5
- data/bin/rtlr.bat +0 -0
- data/features/command_line/dest_option.feature +34 -0
- data/features/command_line/output_option.feature +31 -0
- data/features/command_line/parser_generator.feature +80 -0
- data/features/grammar/word_literal.feature +3 -3
- data/features/step_definitions/cli_steps.rb +3 -0
- data/features/step_definitions/{rattler_steps.rb → grammar_steps.rb} +0 -0
- data/features/support/env.rb +1 -0
- data/lib/rattler.rb +2 -2
- data/lib/rattler/back_end/parser_generator.rb +1 -0
- data/lib/rattler/back_end/parser_generator/one_or_more_generator.rb +17 -28
- data/lib/rattler/back_end/parser_generator/repeat_generating.rb +69 -0
- data/lib/rattler/back_end/parser_generator/rule_generator.rb +1 -1
- data/lib/rattler/back_end/parser_generator/zero_or_more_generator.rb +9 -63
- data/lib/rattler/grammar/grammar_parser.rb +1 -1
- data/lib/rattler/grammar/metagrammar.rb +63 -73
- data/lib/rattler/grammar/rattler.rtlr +5 -13
- data/lib/rattler/parsers/combining.rb +7 -0
- data/lib/rattler/parsers/match_joining.rb +7 -0
- data/lib/rattler/parsers/predicate.rb +7 -0
- data/lib/rattler/parsers/sequence.rb +1 -1
- data/lib/rattler/parsers/token.rb +1 -1
- data/lib/rattler/runner.rb +6 -0
- data/lib/rattler/runtime.rb +1 -1
- data/lib/rattler/runtime/extended_packrat_parser.rb +62 -0
- data/lib/rattler/runtime/packrat_parser.rb +46 -12
- data/lib/rattler/runtime/parse_failure.rb +1 -1
- data/lib/rattler/util/parser_spec_helper.rb +1 -0
- data/spec/rattler/back_end/parser_generator/one_or_more_generator_spec.rb +14 -4
- data/spec/rattler/back_end/parser_generator/zero_or_more_generator_spec.rb +12 -2
- data/spec/rattler/runtime/extended_packrat_parser_spec.rb +36 -0
- data/spec/rattler/runtime/packrat_parser_spec.rb +1 -1
- data/spec/rattler/runtime/shared_parser_examples.rb +29 -0
- metadata +462 -79
- data/lib/rattler/runtime/wdm_parser.rb +0 -83
- data/spec/rattler/runtime/wdm_parser_spec.rb +0 -21
data/README.rdoc
CHANGED
@@ -14,6 +14,8 @@ A language syntax is specified in a grammar using the Rattler syntax. Parser
|
|
14
14
|
classes and modules can be generated statically using the "rtlr" command or
|
15
15
|
dynamically from strings.
|
16
16
|
|
17
|
+
{RDoc}[http://rubydoc.info/gems/rattler/0.2.0/frames]
|
18
|
+
|
17
19
|
== FEATURES:
|
18
20
|
|
19
21
|
* Uses readable PEG-like grammars
|
data/bin/rtlr
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
|
3
|
+
rattler_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
|
+
$:.unshift rattler_dir unless $:.include? rattler_dir
|
5
|
+
|
6
|
+
require 'rattler/runner'
|
7
|
+
|
8
|
+
::Rattler::Runner.run(ARGV.clone)
|
data/bin/rtlr.bat
CHANGED
File without changes
|
@@ -0,0 +1,34 @@
|
|
1
|
+
Feature: --dest option
|
2
|
+
|
3
|
+
Use the --dest (or -d) option to specify a different (base) destination
|
4
|
+
directory.
|
5
|
+
|
6
|
+
Scenario: Grammar as top-level module
|
7
|
+
Given a file named "binary.rtlr" with:
|
8
|
+
"""
|
9
|
+
grammar BinaryGrammar
|
10
|
+
expr <- [01]*
|
11
|
+
"""
|
12
|
+
And a directory named "lib"
|
13
|
+
When I run "rtlr --dest lib binary.rtlr"
|
14
|
+
Then the output should contain "binary.rtlr -> lib/binary_grammar.rb"
|
15
|
+
And the file "lib/binary_grammar.rb" should contain:
|
16
|
+
"""
|
17
|
+
module BinaryGrammar
|
18
|
+
"""
|
19
|
+
|
20
|
+
Scenario: Grammar as top-level module
|
21
|
+
Given a file named "binary.rtlr" with:
|
22
|
+
"""
|
23
|
+
grammar Examples::BinaryGrammar
|
24
|
+
expr <- [01]*
|
25
|
+
"""
|
26
|
+
And a directory named "lib/examples"
|
27
|
+
When I run "rtlr --dest lib binary.rtlr"
|
28
|
+
Then the output should contain "binary.rtlr -> lib/examples/binary_grammar.rb"
|
29
|
+
And the file "lib/examples/binary_grammar.rb" should contain:
|
30
|
+
"""
|
31
|
+
module Examples
|
32
|
+
# @private
|
33
|
+
module BinaryGrammar
|
34
|
+
"""
|
@@ -0,0 +1,31 @@
|
|
1
|
+
Feature: --output option
|
2
|
+
|
3
|
+
Use the --output (or -o) option to specify a different output file name, or
|
4
|
+
"-" for STDOUT.
|
5
|
+
|
6
|
+
Scenario: Specify a different file name
|
7
|
+
Given a file named "binary.rtlr" with:
|
8
|
+
"""
|
9
|
+
grammar BinaryGrammar
|
10
|
+
expr <- [01]*
|
11
|
+
"""
|
12
|
+
And a directory named "lib"
|
13
|
+
When I run "rtlr --output my_grammar.rb binary.rtlr"
|
14
|
+
Then the output should contain "binary.rtlr -> my_grammar.rb"
|
15
|
+
And the file "my_grammar.rb" should contain:
|
16
|
+
"""
|
17
|
+
module BinaryGrammar
|
18
|
+
"""
|
19
|
+
|
20
|
+
Scenario: Use "-" to write to STDOUT
|
21
|
+
Given a file named "binary.rtlr" with:
|
22
|
+
"""
|
23
|
+
grammar BinaryGrammar
|
24
|
+
expr <- [01]*
|
25
|
+
"""
|
26
|
+
And a directory named "lib"
|
27
|
+
When I run "rtlr --output - binary.rtlr"
|
28
|
+
Then the output should contain:
|
29
|
+
"""
|
30
|
+
module BinaryGrammar
|
31
|
+
"""
|
@@ -0,0 +1,80 @@
|
|
1
|
+
Feature: Parser Generator
|
2
|
+
|
3
|
+
The "rtlr" command is used to generate a grammar module or parser class from
|
4
|
+
a Rattler grammar file.
|
5
|
+
|
6
|
+
Scenario: Getting help
|
7
|
+
When I run "rtlr --help"
|
8
|
+
Then the output should contain:
|
9
|
+
"""
|
10
|
+
Usage: rtlr FILENAME [options]
|
11
|
+
|
12
|
+
-d, --dest DIRECTORY Specify the destination directory
|
13
|
+
-o, --output FILENAME Specify a different output filename
|
14
|
+
-f, --force Force overwrite if the output file exists
|
15
|
+
|
16
|
+
-h, --help Show this message
|
17
|
+
"""
|
18
|
+
|
19
|
+
Scenario: Generate a grammar module
|
20
|
+
Given a file named "binary.rtlr" with:
|
21
|
+
"""
|
22
|
+
grammar BinaryGrammar
|
23
|
+
expr <- [01]*
|
24
|
+
"""
|
25
|
+
When I run "rtlr binary.rtlr"
|
26
|
+
Then the output should contain "binary.rtlr -> binary_grammar.rb"
|
27
|
+
And the file "binary_grammar.rb" should contain:
|
28
|
+
"""
|
29
|
+
# @private
|
30
|
+
module BinaryGrammar #:nodoc:
|
31
|
+
|
32
|
+
# @private
|
33
|
+
def start_rule #:nodoc:
|
34
|
+
:expr
|
35
|
+
end
|
36
|
+
|
37
|
+
# @private
|
38
|
+
def match_expr #:nodoc:
|
39
|
+
a0 = []
|
40
|
+
while r = @scanner.scan(/[01]/)
|
41
|
+
a0 << r
|
42
|
+
end
|
43
|
+
a0
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
"""
|
48
|
+
|
49
|
+
Scenario: Generate a parser class
|
50
|
+
Given a file named "binary.rtlr" with:
|
51
|
+
"""
|
52
|
+
require "rattler"
|
53
|
+
parser BinaryParser < Rattler::Runtime::PackratParser
|
54
|
+
expr <- [01]*
|
55
|
+
"""
|
56
|
+
When I run "rtlr binary.rtlr"
|
57
|
+
Then the output should contain "binary.rtlr -> binary_parser.rb"
|
58
|
+
And the file "binary_parser.rb" should contain:
|
59
|
+
"""
|
60
|
+
require "rattler"
|
61
|
+
|
62
|
+
# @private
|
63
|
+
class BinaryParser < Rattler::Runtime::PackratParser #:nodoc:
|
64
|
+
|
65
|
+
# @private
|
66
|
+
def start_rule #:nodoc:
|
67
|
+
:expr
|
68
|
+
end
|
69
|
+
|
70
|
+
# @private
|
71
|
+
def match_expr #:nodoc:
|
72
|
+
a0 = []
|
73
|
+
while r = @scanner.scan(/[01]/)
|
74
|
+
a0 << r
|
75
|
+
end
|
76
|
+
a0
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
"""
|
@@ -2,9 +2,9 @@ Feature: Word Literal Expressions
|
|
2
2
|
|
3
3
|
A word literal is like a string literal but delimited with backquotes (`) and
|
4
4
|
it means to match the literal only if it is not followed by a word character.
|
5
|
-
It is useful
|
6
|
-
character can be defined using the %word_character directive. By default
|
7
|
-
word charactacter is an alphanumeric character or the underscore character.
|
5
|
+
It is useful when you want to match whole words only. What is considered a
|
6
|
+
word character can be defined using the %word_character directive. By default
|
7
|
+
a word charactacter is an alphanumeric character or the underscore character.
|
8
8
|
|
9
9
|
In order to define keywords
|
10
10
|
As a language designer
|
File without changes
|
data/features/support/env.rb
CHANGED
data/lib/rattler.rb
CHANGED
@@ -23,14 +23,14 @@ module Rattler
|
|
23
23
|
|
24
24
|
# @private
|
25
25
|
@@defaults = {
|
26
|
-
:type => :
|
26
|
+
:type => :extended_packrat
|
27
27
|
}
|
28
28
|
|
29
29
|
# @private
|
30
30
|
@@parser_types = {
|
31
31
|
:recursive_descent => :RecursiveDescentParser,
|
32
32
|
:packrat => :PackratParser,
|
33
|
-
:
|
33
|
+
:extended_packrat => :ExtendedPackratParser
|
34
34
|
}
|
35
35
|
|
36
36
|
# Define parse rules with the given block
|
@@ -38,6 +38,7 @@ module Rattler::BackEnd
|
|
38
38
|
autoload :SkipGenerator, 'rattler/back_end/parser_generator/skip_generator'
|
39
39
|
autoload :LabelGenerator, 'rattler/back_end/parser_generator/label_generator'
|
40
40
|
autoload :FailGenerator, 'rattler/back_end/parser_generator/fail_generator'
|
41
|
+
autoload :RepeatGenerating, 'rattler/back_end/parser_generator/repeat_generating'
|
41
42
|
autoload :PredicatePropogating, 'rattler/back_end/parser_generator/predicate_propogating'
|
42
43
|
autoload :TokenPropogating, 'rattler/back_end/parser_generator/token_propogating'
|
43
44
|
autoload :SkipPropogating, 'rattler/back_end/parser_generator/skip_propogating'
|
@@ -3,29 +3,10 @@ require 'rattler/back_end/parser_generator'
|
|
3
3
|
module Rattler::BackEnd::ParserGenerator
|
4
4
|
|
5
5
|
# @private
|
6
|
-
class OneOrMoreGenerator <
|
6
|
+
class OneOrMoreGenerator < ExprGenerator #:nodoc:
|
7
|
+
include RepeatGenerating
|
7
8
|
include PredicatePropogating
|
8
9
|
|
9
|
-
def gen_basic_top_level(one_or_more)
|
10
|
-
if one_or_more.capturing?
|
11
|
-
required_match { super }
|
12
|
-
else
|
13
|
-
gen_skip_top_level one_or_more
|
14
|
-
end
|
15
|
-
end
|
16
|
-
|
17
|
-
def gen_dispatch_action_top_level(one_or_more, target, method_name)
|
18
|
-
required_match { super }
|
19
|
-
end
|
20
|
-
|
21
|
-
def gen_direct_action_top_level(one_or_more, action)
|
22
|
-
required_match { super }
|
23
|
-
end
|
24
|
-
|
25
|
-
def gen_token_top_level(one_or_more)
|
26
|
-
required_match { super }
|
27
|
-
end
|
28
|
-
|
29
10
|
def gen_skip_top_level(one_or_more)
|
30
11
|
(@g << "#{result_name} = false").newline
|
31
12
|
@g << 'while '
|
@@ -34,13 +15,12 @@ module Rattler::BackEnd::ParserGenerator
|
|
34
15
|
@g << result_name
|
35
16
|
end
|
36
17
|
|
37
|
-
|
38
|
-
|
39
|
-
def
|
40
|
-
|
41
|
-
@g << " unless #{accumulator_name}.empty?"
|
18
|
+
protected
|
19
|
+
|
20
|
+
def gen_result(captures)
|
21
|
+
@g << captures << " unless #{accumulator_name}.empty?"
|
42
22
|
end
|
43
|
-
|
23
|
+
|
44
24
|
end
|
45
25
|
|
46
26
|
# @private
|
@@ -56,7 +36,16 @@ module Rattler::BackEnd::ParserGenerator
|
|
56
36
|
# @private
|
57
37
|
class TopLevelOneOrMoreGenerator < OneOrMoreGenerator #:nodoc:
|
58
38
|
include TopLevel
|
59
|
-
include
|
39
|
+
include NestedGenerators
|
40
|
+
|
41
|
+
def gen_assert(parser)
|
42
|
+
generate parser.child, :gen_assert_top_level
|
43
|
+
end
|
44
|
+
|
45
|
+
def gen_disallow(parser)
|
46
|
+
generate parser.child, :gen_disallow_top_level
|
47
|
+
end
|
48
|
+
|
60
49
|
end
|
61
50
|
|
62
51
|
def OneOrMoreGenerator.top_level(*args)
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'rattler/back_end/parser_generator'
|
2
|
+
|
3
|
+
module Rattler::BackEnd::ParserGenerator
|
4
|
+
# @private
|
5
|
+
module RepeatGenerating #:nodoc:
|
6
|
+
|
7
|
+
def gen_basic_nested(repeat)
|
8
|
+
atomic_block { gen_basic_top_level repeat }
|
9
|
+
end
|
10
|
+
|
11
|
+
def gen_basic_top_level(repeat)
|
12
|
+
if repeat.capturing?
|
13
|
+
gen_loop repeat, :gen_basic_nested
|
14
|
+
gen_result accumulator_name
|
15
|
+
else
|
16
|
+
gen_skip_top_level repeat
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def gen_dispatch_action_nested(repeat, target, method_name)
|
21
|
+
atomic_block do
|
22
|
+
gen_dispatch_action_top_level repeat, target, method_name
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def gen_dispatch_action_top_level(repeat, target, method_name)
|
27
|
+
gen_loop repeat
|
28
|
+
gen_result dispatch_action_result(target, method_name,
|
29
|
+
:array_expr => "select_captures(#{accumulator_name})")
|
30
|
+
end
|
31
|
+
|
32
|
+
def gen_direct_action_nested(repeat, action)
|
33
|
+
atomic_block { gen_direct_action_top_level repeat, action }
|
34
|
+
end
|
35
|
+
|
36
|
+
def gen_direct_action_top_level(repeat, action)
|
37
|
+
gen_loop repeat
|
38
|
+
gen_result direct_action_result(action,
|
39
|
+
:bind_args => ["select_captures(#{accumulator_name})"])
|
40
|
+
end
|
41
|
+
|
42
|
+
def gen_token_nested(repeat)
|
43
|
+
atomic_block { gen_token_top_level repeat }
|
44
|
+
end
|
45
|
+
|
46
|
+
def gen_token_top_level(repeat)
|
47
|
+
gen_loop repeat, :gen_token
|
48
|
+
gen_result "#{accumulator_name}.join"
|
49
|
+
end
|
50
|
+
|
51
|
+
def gen_skip_nested(repeat)
|
52
|
+
atomic_block { gen_skip_top_level repeat }
|
53
|
+
end
|
54
|
+
|
55
|
+
private
|
56
|
+
|
57
|
+
def gen_loop(repeat, gen_method = :gen_basic)
|
58
|
+
(@g << "#{accumulator_name} = []").newline
|
59
|
+
@g << "while #{result_name} = "
|
60
|
+
generate repeat.child, gen_method
|
61
|
+
@g.block('') { @g << "#{accumulator_name} << #{result_name}" }.newline
|
62
|
+
end
|
63
|
+
|
64
|
+
def accumulator_name
|
65
|
+
"a#{repeat_level}"
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
@@ -80,7 +80,7 @@ module Rattler::BackEnd::ParserGenerator
|
|
80
80
|
def gen_parser_def(parser_name, base_name)
|
81
81
|
nest_modules(parser_name.split('::')) do |name|
|
82
82
|
(@g << "# @private").newline
|
83
|
-
module_block("class #{name} < #{base_name}") { yield }
|
83
|
+
module_block("class #{name} < #{base_name} #:nodoc:") { yield }
|
84
84
|
end
|
85
85
|
end
|
86
86
|
|
@@ -4,19 +4,7 @@ module Rattler::BackEnd::ParserGenerator
|
|
4
4
|
|
5
5
|
# @private
|
6
6
|
class ZeroOrMoreGenerator < ExprGenerator #:nodoc:
|
7
|
-
|
8
|
-
def gen_basic_nested(zero_or_more)
|
9
|
-
atomic_block { gen_basic_top_level zero_or_more }
|
10
|
-
end
|
11
|
-
|
12
|
-
def gen_basic_top_level(zero_or_more)
|
13
|
-
if zero_or_more.capturing?
|
14
|
-
gen_loop(zero_or_more)
|
15
|
-
@g << accumulator_name
|
16
|
-
else
|
17
|
-
gen_skip_top_level zero_or_more
|
18
|
-
end
|
19
|
-
end
|
7
|
+
include RepeatGenerating
|
20
8
|
|
21
9
|
def gen_assert(optional)
|
22
10
|
@g << 'true'
|
@@ -26,61 +14,19 @@ module Rattler::BackEnd::ParserGenerator
|
|
26
14
|
@g << 'false'
|
27
15
|
end
|
28
16
|
|
29
|
-
def
|
30
|
-
atomic_block do
|
31
|
-
gen_dispatch_action_top_level zero_or_more, target, method_name
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
def gen_dispatch_action_top_level(zero_or_more, target, method_name)
|
36
|
-
gen_loop zero_or_more
|
37
|
-
@g << dispatch_action_result(target, method_name,
|
38
|
-
:array_expr => "select_captures(#{accumulator_name})")
|
39
|
-
end
|
40
|
-
|
41
|
-
def gen_direct_action_nested(zero_or_more, action)
|
42
|
-
atomic_block { gen_direct_action_top_level zero_or_more, action }
|
43
|
-
end
|
44
|
-
|
45
|
-
def gen_direct_action_top_level(zero_or_more, action)
|
46
|
-
gen_loop zero_or_more
|
47
|
-
@g << direct_action_result(action,
|
48
|
-
:bind_args => ["select_captures(#{accumulator_name})"])
|
49
|
-
end
|
50
|
-
|
51
|
-
def gen_token_nested(zero_or_more)
|
52
|
-
atomic_block { gen_token_top_level zero_or_more }
|
53
|
-
end
|
54
|
-
|
55
|
-
def gen_token_top_level(zero_or_more)
|
56
|
-
gen_loop zero_or_more, :gen_token
|
57
|
-
@g << "#{accumulator_name}.join"
|
58
|
-
end
|
59
|
-
|
60
|
-
def gen_skip_nested(zero_or_more)
|
61
|
-
atomic_block { gen_skip_top_level zero_or_more }
|
62
|
-
end
|
63
|
-
|
64
|
-
def gen_skip_top_level(zero_or_more)
|
17
|
+
def gen_skip_top_level(repeat)
|
65
18
|
@g << 'while '
|
66
|
-
generate
|
19
|
+
generate repeat.child, :gen_intermediate_skip
|
67
20
|
(@g << '; end').newline
|
68
21
|
@g << 'true'
|
69
22
|
end
|
70
23
|
|
71
|
-
|
72
|
-
|
73
|
-
def
|
74
|
-
|
75
|
-
@g << "while #{result_name} = "
|
76
|
-
generate zero_or_more.child, gen_method
|
77
|
-
@g.block('') { @g << "#{accumulator_name} << #{result_name}" }.newline
|
78
|
-
end
|
79
|
-
|
80
|
-
def accumulator_name
|
81
|
-
"a#{repeat_level}"
|
24
|
+
protected
|
25
|
+
|
26
|
+
def gen_result(captures)
|
27
|
+
@g << captures
|
82
28
|
end
|
83
|
-
|
29
|
+
|
84
30
|
end
|
85
31
|
|
86
32
|
# @private
|
@@ -96,7 +42,7 @@ module Rattler::BackEnd::ParserGenerator
|
|
96
42
|
# @private
|
97
43
|
class TopLevelZeroOrMoreGenerator < ZeroOrMoreGenerator #:nodoc:
|
98
44
|
include TopLevel
|
99
|
-
include
|
45
|
+
include NestedGenerators
|
100
46
|
end
|
101
47
|
|
102
48
|
def ZeroOrMoreGenerator.top_level(*args)
|