treetop 1.5.3 → 1.6.12

Sign up to get free protection for your applications and to get access to all the features.
Files changed (95) hide show
  1. checksums.yaml +5 -13
  2. data/Gemfile +12 -0
  3. data/History.txt +18 -0
  4. data/README.md +4 -0
  5. data/Rakefile +20 -40
  6. data/Treetop.tmbundle/Preferences/Comments.tmPreferences +28 -0
  7. data/Treetop.tmbundle/Snippets/grammar ___ end.tmSnippet +20 -0
  8. data/Treetop.tmbundle/Snippets/rule ___ end.tmSnippet +18 -0
  9. data/Treetop.tmbundle/Support/nibs/SyntaxTreeViewer.nib/designable.nib +1524 -0
  10. data/Treetop.tmbundle/Support/nibs/SyntaxTreeViewer.nib/keyedobjects.nib +0 -0
  11. data/Treetop.tmbundle/Support/syntax_tree_viewer.rb +117 -0
  12. data/Treetop.tmbundle/Syntaxes/Treetop Grammar.tmLanguage +358 -0
  13. data/Treetop.tmbundle/info.plist +10 -0
  14. data/doc/pitfalls_and_advanced_techniques.markdown +7 -1
  15. data/doc/syntactic_recognition.markdown +7 -2
  16. data/doc/tt.1 +1 -1
  17. data/examples/indented_blocks/indented_blocks.tt +73 -0
  18. data/examples/indented_blocks/indented_blocks_test.rb +24 -0
  19. data/lib/treetop/compiler/grammar_compiler.rb +6 -3
  20. data/lib/treetop/compiler/metagrammar.rb +301 -159
  21. data/lib/treetop/compiler/metagrammar.treetop +96 -13
  22. data/lib/treetop/compiler/node_classes/anything_symbol.rb +10 -2
  23. data/lib/treetop/compiler/node_classes/atomic_expression.rb +2 -2
  24. data/lib/treetop/compiler/node_classes/character_class.rb +10 -2
  25. data/lib/treetop/compiler/node_classes/choice.rb +11 -7
  26. data/lib/treetop/compiler/node_classes/nonterminal.rb +6 -2
  27. data/lib/treetop/compiler/node_classes/parenthesized_expression.rb +5 -1
  28. data/lib/treetop/compiler/node_classes/parsing_expression.rb +10 -1
  29. data/lib/treetop/compiler/node_classes/parsing_rule.rb +1 -1
  30. data/lib/treetop/compiler/node_classes/predicate.rb +8 -1
  31. data/lib/treetop/compiler/node_classes/predicate_block.rb +7 -0
  32. data/lib/treetop/compiler/node_classes/repetition.rb +28 -8
  33. data/lib/treetop/compiler/node_classes/sequence.rb +5 -1
  34. data/lib/treetop/compiler/node_classes/terminal.rb +36 -22
  35. data/lib/treetop/compiler/ruby_builder.rb +2 -2
  36. data/lib/treetop/ruby_extensions/string.rb +0 -6
  37. data/lib/treetop/runtime/compiled_parser.rb +33 -14
  38. data/lib/treetop/runtime/syntax_node.rb +24 -15
  39. data/lib/treetop/runtime/terminal_parse_failure.rb +4 -3
  40. data/lib/treetop/runtime/terminal_syntax_node.rb +4 -4
  41. data/lib/treetop/version.rb +2 -2
  42. data/treetop.gemspec +25 -165
  43. metadata +43 -98
  44. data/doc/site.rb +0 -112
  45. data/doc/sitegen.rb +0 -65
  46. data/examples/lambda_calculus/lambda_calculus +0 -0
  47. data/spec/compiler/and_predicate_spec.rb +0 -36
  48. data/spec/compiler/anything_symbol_spec.rb +0 -44
  49. data/spec/compiler/character_class_spec.rb +0 -301
  50. data/spec/compiler/choice_spec.rb +0 -80
  51. data/spec/compiler/circular_compilation_spec.rb +0 -30
  52. data/spec/compiler/failure_propagation_functional_spec.rb +0 -21
  53. data/spec/compiler/grammar_compiler_spec.rb +0 -113
  54. data/spec/compiler/grammar_spec.rb +0 -41
  55. data/spec/compiler/multibyte_chars_spec.rb +0 -38
  56. data/spec/compiler/namespace_spec.rb +0 -42
  57. data/spec/compiler/nonterminal_symbol_spec.rb +0 -40
  58. data/spec/compiler/not_predicate_spec.rb +0 -38
  59. data/spec/compiler/occurrence_range_spec.rb +0 -189
  60. data/spec/compiler/one_or_more_spec.rb +0 -35
  61. data/spec/compiler/optional_spec.rb +0 -37
  62. data/spec/compiler/parenthesized_expression_spec.rb +0 -19
  63. data/spec/compiler/parsing_rule_spec.rb +0 -61
  64. data/spec/compiler/repeated_subrule_spec.rb +0 -29
  65. data/spec/compiler/semantic_predicate_spec.rb +0 -175
  66. data/spec/compiler/sequence_spec.rb +0 -129
  67. data/spec/compiler/terminal_spec.rb +0 -170
  68. data/spec/compiler/terminal_symbol_spec.rb +0 -37
  69. data/spec/compiler/test_grammar.treetop +0 -7
  70. data/spec/compiler/test_grammar.tt +0 -7
  71. data/spec/compiler/test_grammar_do.treetop +0 -7
  72. data/spec/compiler/test_grammar_magic_coding.treetop +0 -8
  73. data/spec/compiler/test_grammar_magic_encoding.treetop +0 -8
  74. data/spec/compiler/tt_compiler_spec.rb +0 -224
  75. data/spec/compiler/zero_or_more_spec.rb +0 -56
  76. data/spec/composition/a.treetop +0 -11
  77. data/spec/composition/b.treetop +0 -11
  78. data/spec/composition/c.treetop +0 -10
  79. data/spec/composition/d.treetop +0 -10
  80. data/spec/composition/f.treetop +0 -17
  81. data/spec/composition/grammar_composition_spec.rb +0 -40
  82. data/spec/composition/subfolder/e_includes_c.treetop +0 -15
  83. data/spec/ruby_extensions/string_spec.rb +0 -32
  84. data/spec/runtime/compiled_parser_spec.rb +0 -123
  85. data/spec/runtime/interval_skip_list/delete_spec.rb +0 -147
  86. data/spec/runtime/interval_skip_list/expire_range_spec.rb +0 -349
  87. data/spec/runtime/interval_skip_list/insert_and_delete_node_spec.rb +0 -385
  88. data/spec/runtime/interval_skip_list/insert_spec.rb +0 -660
  89. data/spec/runtime/interval_skip_list/interval_skip_list_spec.graffle +0 -6175
  90. data/spec/runtime/interval_skip_list/interval_skip_list_spec.rb +0 -58
  91. data/spec/runtime/interval_skip_list/palindromic_fixture.rb +0 -35
  92. data/spec/runtime/interval_skip_list/palindromic_fixture_spec.rb +0 -163
  93. data/spec/runtime/interval_skip_list/spec_helper.rb +0 -91
  94. data/spec/runtime/syntax_node_spec.rb +0 -77
  95. data/spec/spec_helper.rb +0 -115
@@ -1,58 +0,0 @@
1
- #require 'runtime/interval_skip_list/spec_helper'
2
- #
3
- #MAX_INTERVAL = 100000
4
- #
5
- #describe IntervalSkipList do
6
- # describe "#next_node_height" do
7
- # attr_reader :list
8
- #
9
- # before do
10
- # @list = IntervalSkipList.new
11
- # end
12
- #
13
- # it "returns a number between 1 and the max_height of the list" do
14
- # height = list.next_node_height
15
- # height.should be <= list.max_height
16
- # height.should be > 0
17
- # end
18
- # end
19
- #
20
- # describe "a list with 1000 random intervals" do
21
- # attr_reader :list, :inserted_ranges
22
- #
23
- # before do
24
- # @list = IntervalSkipList.new
25
- # @inserted_ranges = []
26
- #
27
- # 0.upto(10) do |i|
28
- # first, last = [rand(MAX_INTERVAL), rand(MAX_INTERVAL)].sort
29
- # range = first..last
30
- # list.insert(range, i)
31
- # inserted_ranges.push(range)
32
- # end
33
- # end
34
- #
35
- # it "functions correctly for stabbing queries" do
36
- # 10000.times do
37
- # n = rand(MAX_INTERVAL)
38
- # ranges = list.containing(n).sort
39
- #
40
- # expected_ranges = []
41
- # inserted_ranges.each_with_index do |range,i|
42
- # expected_ranges.push(i) if n > range.first && n < range.last
43
- # end
44
- # expected_ranges.sort!
45
- # unless ranges == expected_ranges
46
- # puts "N = #{n}"
47
- # puts "Expected: " + expected_ranges.inspect
48
- # puts "Actual: " + ranges.inspect
49
- # expected_ranges.size.should be <= ranges.size
50
- # puts "Missing containers: #{(expected_ranges.map {|o| o.object_id} - ranges.map {|o| o.object_id}).inspect}"
51
- # puts "Unexpected containers: #{(ranges.map {|o| o.object_id} - expected_ranges.map {|o| o.object_id}).inspect}"
52
- # puts "Inserted Ranges: #{inserted_ranges.inspect}"
53
- # puts "Expected Ranges: #{expected_ranges.map {|i| inserted_ranges[i]}.inspect}"
54
- # end
55
- # end
56
- # end
57
- # end
58
- #end
@@ -1,35 +0,0 @@
1
- module PalindromicFixtureSharedContext
2
- extend RSpec::Core::SharedContext
3
- include IntervalSkipListSpecHelper
4
-
5
- attr_reader :list, :node
6
-
7
- def construct_interval_skip_list
8
- @list = IntervalSkipList.new
9
- end
10
-
11
- def expected_node_heights
12
- [3, 2, 1, 3, 1, 2, 3]
13
- end
14
-
15
- def populate_interval_skip_list
16
- @list.insert(1..3, :a)
17
- @list.insert(1..5, :b)
18
- @list.insert(1..7, :c)
19
- @list.insert(1..9, :d)
20
- @list.insert(1..11, :e)
21
- @list.insert(1..13, :f)
22
- @list.insert(5..13, :g)
23
- end
24
-
25
- def make_it_determinisitic
26
- extend NextNodeHeightIsDeterministicSharedContext # use the method without getting the filter
27
- next_node_height_is_deterministic
28
- end
29
-
30
- before :each do
31
- construct_interval_skip_list
32
- make_it_determinisitic
33
- populate_interval_skip_list
34
- end
35
- end
@@ -1,163 +0,0 @@
1
- require 'runtime/interval_skip_list/spec_helper'
2
-
3
- describe "The palindromic fixture", :palindromic => true do
4
- include PalindromicFixtureSharedContext
5
-
6
- describe " #nodes" do
7
- describe "[0]" do
8
- before do
9
- @node = list.nodes[0]
10
- end
11
-
12
- it "has a key of 1 and a height of 3" do
13
- node.key.should == 1
14
- node.height.should == 3
15
- end
16
-
17
- it "has :c, :d, :e, and :f as its only forward markers at level 2" do
18
- node.forward_markers[2].should have_markers(:c, :d, :e, :f)
19
- end
20
-
21
- it "has :a, :b as its only forward markers at level 1" do
22
- node.forward_markers[1].should have_markers(:a, :b)
23
- end
24
-
25
- it "has no forward markers at level 0" do
26
- node.forward_markers[0].should be_empty
27
- end
28
-
29
- it "has no markers" do
30
- node.markers.should be_empty
31
- end
32
- end
33
-
34
- describe "[1]" do
35
- before do
36
- @node = list.nodes[1]
37
- end
38
-
39
- it "has a key of 3 and a height of 2" do
40
- node.key.should == 3
41
- node.height.should == 2
42
- end
43
-
44
- it "has no forward markers at level 1" do
45
- node.forward_markers[1].should be_empty
46
- end
47
-
48
- it "has :b as its only forward marker at level 0" do
49
- node.forward_markers[0].should have_marker(:b)
50
- end
51
-
52
- it "has :a and :b as its only markers" do
53
- node.markers.should have_markers(:a, :b)
54
- end
55
- end
56
-
57
- describe "[2]" do
58
- before do
59
- @node = list.nodes[2]
60
- end
61
-
62
- it "has a key of 5 and a height of 1" do
63
- node.key.should == 5
64
- node.height.should == 1
65
- end
66
-
67
- it "has :g as its only forward marker at level 0" do
68
- node.forward_markers[0].should have_marker(:g)
69
- end
70
-
71
- it "has :b as its only marker" do
72
- node.markers.should have_marker(:b)
73
- end
74
- end
75
-
76
- describe "[3]" do
77
- before do
78
- @node = list.nodes[3]
79
- end
80
-
81
- it "has a key of 7 and a height of 3" do
82
- node.key.should == 7
83
- node.height.should == 3
84
- end
85
-
86
- it "has :f and :g as its only forward markers at level 2" do
87
- node.forward_markers[2].should have_markers(:f, :g)
88
- end
89
-
90
- it "has :e as its only forward markers at level 1" do
91
- node.forward_markers[1].should have_marker(:e)
92
- end
93
-
94
- it "has :d as its only forward marker at level 0" do
95
- node.forward_markers[0].should have_markers(:d)
96
- end
97
-
98
- it "has :c, :d, :e, :f and :g as its only markers" do
99
- node.markers.should have_markers(:c, :d, :e, :f, :g)
100
- end
101
- end
102
-
103
- describe "[4]" do
104
- before do
105
- @node = list.nodes[4]
106
- end
107
-
108
- it "has a key of 9 and a height of 1" do
109
- node.key.should == 9
110
- node.height.should == 1
111
- end
112
-
113
- it "has no forward markers at any level" do
114
- node.forward_markers[0].should be_empty
115
- end
116
-
117
- it "has :d as its only marker" do
118
- node.markers.should have_markers(:d)
119
- end
120
- end
121
-
122
- describe "[5]" do
123
- before do
124
- @node = list.nodes[5]
125
- end
126
-
127
- it "has a key of 11 and a height of 2" do
128
- node.key.should == 11
129
- node.height.should == 2
130
- end
131
-
132
- it "has no forward markers at any level" do
133
- node.forward_markers[0].should be_empty
134
- node.forward_markers[1].should be_empty
135
- end
136
-
137
- it "has :e as its only marker" do
138
- node.markers.should have_markers(:e)
139
- end
140
- end
141
-
142
- describe "[6]" do
143
- before do
144
- @node = list.nodes[6]
145
- end
146
-
147
- it "has a key of 13 and a height of 3" do
148
- node.key.should == 13
149
- node.height.should == 3
150
- end
151
-
152
- it "has no forward markers at any level" do
153
- node.forward_markers[0].should be_empty
154
- node.forward_markers[1].should be_empty
155
- node.forward_markers[2].should be_empty
156
- end
157
-
158
- it "has :f and :g as its only markers" do
159
- node.markers.should have_markers(:f, :g)
160
- end
161
- end
162
- end
163
- end
@@ -1,91 +0,0 @@
1
- require 'spec_helper'
2
-
3
- class IntervalSkipList
4
- public :insert_node, :delete_node, :nodes, :head, :next_node_height
5
- end
6
-
7
- module NextNodeHeightIsDeterministicSharedContext
8
- extend RSpec::Core::SharedContext
9
-
10
- before :each do
11
- next_node_height_is_deterministic
12
- end
13
-
14
- # @todo we call this in one place other than here. cleanup?
15
- def next_node_height_is_deterministic
16
- node_heights = expected_node_heights.dup
17
- stub(list).next_node_height { node_heights.shift }
18
- end
19
- end
20
-
21
- module IntervalSkipListSpecHelper
22
- def contain_marker(marker)
23
- ContainMarkers.new(list, [marker])
24
- end
25
-
26
- def contain_markers(*markers)
27
- ContainMarkers.new(list, markers)
28
- end
29
-
30
- class ContainMarkers
31
- attr_reader :failure_message
32
-
33
- def initialize(list, expected_markers)
34
- @list = list
35
- @expected_markers = expected_markers
36
- end
37
-
38
- def matches?(target_range)
39
- @target_range = target_range
40
-
41
- @target_range.each do |i|
42
- markers = @list.containing(i)
43
-
44
- @expected_markers.each do |expected_marker|
45
- unless markers.include?(expected_marker)
46
- @failure_message = "Expected #{expected_marker.inspect} to contain #{i}, but it doesn't. #{i} is contained by: #{markers.inspect}."
47
- return false
48
- end
49
- end
50
-
51
- markers.each do |marker|
52
- unless @expected_markers.include?(marker)
53
- @failure_message = "Did not expect #{marker.inspect} to contain #{i}. Only expected #{@expected_markers.inspect}."
54
- return false
55
- end
56
- end
57
- end
58
-
59
- true
60
- end
61
- end
62
-
63
- def have_markers(*markers)
64
- HaveMarkers.new(markers)
65
- end
66
-
67
- def have_marker(marker)
68
- HaveMarkers.new([marker])
69
- end
70
-
71
- class HaveMarkers
72
- def initialize(expected_markers)
73
- @expected_markers = expected_markers
74
- end
75
-
76
- def matches?(target)
77
- @target = target
78
- return false unless @target.size == @expected_markers.size
79
- @expected_markers.each do |expected_marker|
80
- return false unless @target.include?(expected_marker)
81
- end
82
- true
83
- end
84
-
85
- def failure_message
86
- "Expected #{@target.inspect} to include only #{@expected_markers.inspect}"
87
- end
88
- end
89
- end
90
-
91
- require 'runtime/interval_skip_list/palindromic_fixture'
@@ -1,77 +0,0 @@
1
- require 'spec_helper'
2
-
3
- module SyntaxNodeSpec
4
- describe "A new terminal syntax node" do
5
- attr_reader :node
6
-
7
- before do
8
- @node = Runtime::SyntaxNode.new("input", 0...3)
9
- end
10
-
11
- it "reports itself as terminal" do
12
- node.should be_terminal
13
- node.should_not be_nonterminal
14
- end
15
-
16
- it "has a text value based on the input and the interval" do
17
- node.text_value.should == "inp"
18
- end
19
-
20
- it "has itself as its only element" do
21
- node.elements.should be_nil
22
- end
23
- end
24
-
25
- describe "A new nonterminal syntax node" do
26
- attr_reader :node
27
-
28
- before do
29
- @elements = [Runtime::SyntaxNode.new('input', 0...3)]
30
- @node = Runtime::SyntaxNode.new('input', 0...3, @elements)
31
- end
32
-
33
- it "reports itself as nonterminal" do
34
- node.should be_nonterminal
35
- node.should_not be_terminal
36
- end
37
-
38
- it "has a text value based on the input and the interval" do
39
- node.text_value.should == "inp"
40
- end
41
-
42
- it "has the elements with which it was instantiated" do
43
- node.elements.should == @elements
44
- end
45
-
46
- it "sets itself as the parent of its elements" do
47
- node.elements.each do |element|
48
- element.parent.should == node
49
- end
50
- end
51
- end
52
-
53
- describe "A new nonterminal syntax node with all children lazily instantiated" do
54
- attr_reader :node
55
-
56
- it "should lazily instantiate its child nodes" do
57
- @node = Runtime::SyntaxNode.new('input', 0...3, [true, true, true])
58
- node.elements.size.should == 3
59
- node.elements.first.interval.should == (0...1)
60
- node.elements.first.parent.should == node
61
- end
62
-
63
- it "should lazily replace stand-in child nodes around real ones" do
64
- @input = "input"
65
- child1 = Runtime::SyntaxNode.new(@input, 1...2)
66
- child2 = Runtime::SyntaxNode.new(@input, 3...4)
67
- @node = Runtime::SyntaxNode.new(@input, 0...5, [true, child1, true, child2, true])
68
- node.elements.size.should == 5
69
-
70
- node.elements[0].interval.should == (0...1)
71
- node.elements[0].parent.should == node
72
- 0.upto(4) do |index|
73
- node.elements[index].text_value.should == @input[index, 1]
74
- end
75
- end
76
- end
77
- end
data/spec/spec_helper.rb DELETED
@@ -1,115 +0,0 @@
1
- require 'rubygems'
2
- require 'benchmark'
3
- require 'rspec'
4
- require 'rspec/core/shared_context'
5
- require 'polyglot'
6
-
7
- $LOAD_PATH.unshift File.expand_path('../../lib')
8
- require 'treetop'
9
- include Treetop
10
-
11
- module Treetop
12
- module ExampleGroupInstanceMethods
13
- module ClassMethods
14
- attr_accessor :parser_class_under_test
15
-
16
- def testing_expression(expression_under_test)
17
- testing_grammar(%{
18
- grammar Test
19
- rule expression_under_test
20
- }+expression_under_test+%{
21
- end
22
- end
23
- }.tabto(0))
24
- end
25
-
26
- def testing_grammar(grammar_under_test)
27
- grammar_node = parse_with_metagrammar(grammar_under_test.strip, :module_or_grammar)
28
- parser_code = grammar_node.compile
29
- class_eval(parser_code)
30
- self.parser_class_under_test = class_eval(grammar_node.parser_name)
31
- end
32
-
33
- def parse_with_metagrammar(input, root)
34
- parser = Treetop::Compiler::MetagrammarParser.new
35
- parser.root = root
36
- node = parser.parse(input)
37
- raise parser.failure_reason unless node
38
- node
39
- end
40
-
41
- end
42
-
43
- attr_reader :parser
44
-
45
- def parse_with_metagrammar(input, root)
46
- self.class.parse_with_metagrammar(input, root)
47
- end
48
-
49
- def parser_class_under_test
50
- self.class.parser_class_under_test
51
- end
52
-
53
- def parse(input, options = {})
54
- @parser = parser_class_under_test.new
55
- unless options[:consume_all_input].nil?
56
- parser.consume_all_input = options.delete(:consume_all_input)
57
- end
58
- result = parser.parse(input, options)
59
- yield result if block_given?
60
- result
61
- end
62
-
63
- def parse_multibyte(input, options = {})
64
- require 'active_support/all'
65
-
66
- if RUBY_VERSION !~ /^(1\.9|2\.0)/ && 'NONE' == $KCODE then $KCODE = 'UTF8' end
67
- # rspec 1.3 used to do something similar (set it to 'u') that we need
68
- # for activerecord multibyte wrapper to kick in (1.8 only? @todo)
69
-
70
- parse(input.mb_chars, options)
71
- end
72
-
73
- def compiling_grammar(grammar_under_test)
74
- lambda {
75
- grammar_node = parse_with_metagrammar(grammar_under_test.strip, :grammar)
76
- parser_code = grammar_node.compile
77
- [grammar_node, parser_code]
78
- }
79
- end
80
-
81
- def compiling_expression(expression_under_test)
82
- compiling_grammar(%{
83
- grammar Test
84
- rule expression_under_test
85
- #{expression_under_test}
86
- end
87
- end
88
- }.tabto(0))
89
- end
90
-
91
- def optionally_benchmark(&block)
92
- if BENCHMARK
93
- Benchmark.bm do |x|
94
- x.report(&block)
95
- end
96
- else
97
- yield
98
- end
99
- end
100
- end
101
- end
102
-
103
- RSpec.configure do |c|
104
- c.mock_with :rr
105
- c.extend Treetop::ExampleGroupInstanceMethods::ClassMethods
106
- c.include Treetop::ExampleGroupInstanceMethods
107
- end
108
-
109
- class Symbol
110
- def to_proc
111
- lambda do |x|
112
- x.send(self)
113
- end
114
- end unless method_defined?(:to_proc)
115
- end