treetop 1.2.0 → 1.2.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (37) hide show
  1. data/Rakefile +1 -1
  2. data/doc/contributing_and_planned_features.markdown +3 -11
  3. data/doc/index.markdown +65 -4
  4. data/doc/semantic_interpretation.markdown +3 -1
  5. data/doc/site.rb +79 -10
  6. data/doc/site/contribute.html +118 -0
  7. data/doc/{images/middle_backgound.png → site/images/bottom_background.png} +0 -0
  8. data/doc/{images → site/images}/middle_background.png +0 -0
  9. data/doc/{images → site/images}/paren_language_output.png +0 -0
  10. data/doc/site/images/pivotal.gif +0 -0
  11. data/doc/site/images/top_background.png +0 -0
  12. data/doc/site/index.html +102 -0
  13. data/doc/site/pitfalls_and_advanced_techniques.html +68 -0
  14. data/doc/site/screen.css +129 -0
  15. data/doc/site/semantic_interpretation.html +214 -0
  16. data/doc/site/syntactic_recognition.html +142 -0
  17. data/doc/site/using_in_ruby.html +34 -0
  18. data/doc/sitegen.rb +60 -0
  19. data/doc/syntactic_recognition.markdown +11 -14
  20. data/doc/using_in_ruby.markdown +7 -3
  21. data/lib/treetop/compiler/metagrammar.rb +2 -2
  22. data/lib/treetop/compiler/metagrammar.treetop +3 -3
  23. data/lib/treetop/compiler/node_classes.rb +1 -0
  24. data/lib/treetop/compiler/node_classes/character_class.rb +5 -1
  25. data/lib/treetop/compiler/node_classes/predicate.rb +1 -1
  26. data/lib/treetop/compiler/node_classes/transient_prefix.rb +9 -0
  27. data/lib/treetop/runtime.rb +2 -1
  28. data/lib/treetop/runtime/interval_skip_list.rb +4 -0
  29. data/lib/treetop/runtime/interval_skip_list/head_node.rb +15 -0
  30. data/lib/treetop/runtime/interval_skip_list/interval_skip_list.rb +200 -0
  31. data/lib/treetop/runtime/interval_skip_list/node.rb +164 -0
  32. data/lib/treetop/runtime/syntax_node.rb +40 -40
  33. metadata +23 -10
  34. data/doc/images/bottom_background.png +0 -0
  35. data/doc/images/top_background.png +0 -0
  36. data/doc/screen.css +0 -52
  37. data/doc/site.html +0 -34
@@ -0,0 +1,34 @@
1
+ <html><head><link rel="stylesheet" href="./screen.css" type="text/css"></link>
2
+ <script src="http://www.google-analytics.com/urchin.js" type="text/javascript">
3
+ </script>
4
+ <script type="text/javascript">
5
+ _uacct = "UA-3418876-1";
6
+ urchinTracker();
7
+ </script>
8
+ </head><body><div id="top"><div id="main_navigation"><ul><li>Documentation</li><li><a href="contribute.html">Contribute</a></li><li><a href="index.html">Home</a></li></ul></div></div><div id="middle"><div id="content"><div id="secondary_navigation"><ul><li><a href="syntactic_recognition.html">Syntax</a></li><li><a href="semantic_interpretation.html">Semantics</a></li><li>Using In Ruby</li><li><a href="pitfalls_and_advanced_techniques.html">Advanced Techniques</a></li></ul></div><div id="documentation_content"><h1>Using Treetop Grammars in Ruby</h1>
9
+
10
+ <h2>Using the Command Line Compiler</h2>
11
+
12
+ <p>You can <code>.treetop</code> files into Ruby source code with the <code>tt</code> command line script. <code>tt</code> takes an list of files with a <code>.treetop</code> extension and compiles them into <code>.rb</code> files of the same name. You can then <code>require</code> these files like any other Ruby script. Alternately, you can supply just one <code>.treetop</code> file and a <code>-o</code> flag to name specify the name of the output file. Improvements to this compilation script are welcome.</p>
13
+
14
+ <pre><code>tt foo.treetop bar.treetop
15
+ tt foo.treetop -o foogrammar.rb
16
+ </code></pre>
17
+
18
+ <h2>Loading A Grammar Directly</h2>
19
+
20
+ <p>The Polyglot gem makes it possible to load <code>.treetop</code> or <code>.tt</code> files directly with <code>require</code>. This will invoke <code>Treetop.load</code>, which automatically compiles the grammar to Ruby and then evaluates the Ruby source. If you are getting errors in methods you define on the syntax tree, try using the command line compiler for better stack trace feedback. A better solution to this issue is in the works.</p>
21
+
22
+ <h2>Instantiating and Using Parsers</h2>
23
+
24
+ <p>If a grammar by the name of <code>Foo</code> is defined, the compiled Ruby source will define a <code>FooParser</code> class. To parse input, create an instance and call its <code>parse</code> method with a string. The parser will return the syntax tree of the match or <code>nil</code> if there is a failure.</p>
25
+
26
+ <pre><code>Treetop.load "arithmetic"
27
+
28
+ parser = ArithmeticParser.new
29
+ if parser.parse('1+1')
30
+ puts 'success'
31
+ else
32
+ puts 'failure'
33
+ end
34
+ </code></pre></div></div></div><div id="bottom"></div></body></html>
data/doc/sitegen.rb ADDED
@@ -0,0 +1,60 @@
1
+ class Layout < Erector::Widget
2
+
3
+ class << self
4
+ def inherited(page_class)
5
+ puts page_class
6
+ (@@page_classes ||= []) << page_class
7
+ end
8
+
9
+ def generate_site
10
+ @@page_classes.each do |page_class|
11
+ page_class.generate_html unless page_class.abstract?
12
+ puts page_class
13
+ end
14
+ end
15
+
16
+ def generate_html
17
+ File.open(absolute_path, 'w') do |file|
18
+ file.write(new.render)
19
+ end
20
+ end
21
+
22
+ def absolute_path
23
+ absolutize(relative_path)
24
+ end
25
+
26
+ def relative_path
27
+ "#{name.gsub('::', '_').underscore}.html"
28
+ end
29
+
30
+ def absolutize(relative_path)
31
+ File.join(File.dirname(__FILE__), "site", relative_path)
32
+ end
33
+
34
+ def abstract
35
+ @abstract = true
36
+ end
37
+
38
+ def abstract?
39
+ @abstract
40
+ end
41
+ end
42
+
43
+ def bluecloth(relative_path)
44
+ File.open(File.join(File.dirname(__FILE__), relative_path)) do |file|
45
+ text BlueCloth.new(file.read).to_html
46
+ end
47
+ end
48
+
49
+ def absolutize(relative_path)
50
+ self.class.absolutize(relative_path)
51
+ end
52
+
53
+ def link_to(link_text, page_class, section_class=nil)
54
+ if instance_of?(page_class) || section_class && is_a?(section_class)
55
+ text link_text
56
+ else
57
+ a link_text, :href => page_class.relative_path
58
+ end
59
+ end
60
+ end
@@ -1,5 +1,5 @@
1
1
  #Syntactic Recognition
2
- Treetop grammars are written in a custom language based on parsing expression grammars. Literature on the subject of parsing expression grammars is useful in writing Treetop grammars.
2
+ Treetop grammars are written in a custom language based on parsing expression grammars. Literature on the subject of <a href="http://en.wikipedia.org/wiki/Parsing_expression_grammar">parsing expression grammars</a> is useful in writing Treetop grammars.
3
3
 
4
4
  #Grammar Structure
5
5
  Treetop grammars look like this:
@@ -25,24 +25,23 @@ The main keywords are:
25
25
  #Parsing Expressions
26
26
  Each rule associates a name with a _parsing expression_. Parsing expressions are a generalization of vanilla regular expressions. Their key feature is the ability to reference other expressions in the grammar by name.
27
27
 
28
- ##Atomic Expressions
29
- ###Terminal Symbols
30
- ####Strings
28
+ ##Terminal Symbols
29
+ ###Strings
31
30
  Strings are surrounded in double or single quotes and must be matched exactly.
32
31
 
33
32
  * `"foo"`
34
33
  * `'foo'`
35
34
 
36
- ####Character Classes
35
+ ###Character Classes
37
36
  Character classes are surrounded by brackets. Their semantics are identical to those used in Ruby's regular expressions.
38
37
 
39
38
  * `[a-zA-Z]`
40
39
  * `[0-9]`
41
40
 
42
- ####The Anything Symbol
41
+ ###The Anything Symbol
43
42
  The anything symbol is represented by a dot (`.`) and matches any single character.
44
43
 
45
- ###Nonterminal Symbols
44
+ ##Nonterminal Symbols
46
45
  Nonterminal symbols are unquoted references to other named rules. They are equivalent to an inline substitution of the named expression.
47
46
 
48
47
  rule foo
@@ -59,32 +58,30 @@ The above grammar is equivalent to:
59
58
  "the dog jumped"
60
59
  end
61
60
 
62
- ##Composite Expressions
63
- ###Ordered Choice
61
+ ##Ordered Choice
64
62
  Parsers attempt to match ordered choices in left-to-right order, and stop after the first successful match.
65
63
 
66
64
  "foobar" / "foo" / "bar"
67
65
 
68
66
  Note that if `"foo"` in the above expression came first, `"foobar"` would never be matched.
69
67
 
70
- ###Sequences
68
+ ##Sequences
71
69
 
72
70
  Sequences are a space-separated list of parsing expressions. They have higher precedence than choices, so choices must be parenthesized to be used as the elements of a sequence.
73
71
 
74
72
  "foo" "bar" ("baz" / "bop")
75
73
 
76
- ###Repetitions
77
- ####Zero or More
74
+ ##Zero or More
78
75
  Parsers will greedily match an expression zero or more times if it is followed by the star (`*`) symbol.
79
76
 
80
77
  * `'foo'*` matches the empty string, `"foo"`, `"foofoo"`, etc.
81
78
 
82
- ####One or More
79
+ ##One or More
83
80
  Parsers will greedily match an expression one or more times if it is followed by the star (`+`) symbol.
84
81
 
85
82
  * `'foo'+` does not match the empty string, but matches `"foo"`, `"foofoo"`, etc.
86
83
 
87
- ###Optional Expressions
84
+ ##Optional Expressions
88
85
  An expression can be declared optional by following it with a question mark (`?`).
89
86
 
90
87
  * `'foo'?` matches `"foo"` or the empty string.
@@ -6,12 +6,16 @@ You can `.treetop` files into Ruby source code with the `tt` command line script
6
6
  tt foo.treetop -o foogrammar.rb
7
7
 
8
8
  ##Loading A Grammar Directly
9
- The `Treetop.load` method takes the path to a `.treetop` file (where the extension is optional), and automatically compiles and evaluates the Ruby source. If you are getting errors in methods you define on the syntax tree, try using the command line compiler for better stack trace feedback. The need to do this is being addressed.
9
+ The Polyglot gem makes it possible to load `.treetop` or `.tt` files directly with `require`. This will invoke `Treetop.load`, which automatically compiles the grammar to Ruby and then evaluates the Ruby source. If you are getting errors in methods you define on the syntax tree, try using the command line compiler for better stack trace feedback. A better solution to this issue is in the works.
10
10
 
11
11
  ##Instantiating and Using Parsers
12
- If a grammar by the name of `Foo` is defined, the compiled Ruby source will define a `FooParser` class. To parse input, create an instance and call its `parse` method with a string.
12
+ If a grammar by the name of `Foo` is defined, the compiled Ruby source will define a `FooParser` class. To parse input, create an instance and call its `parse` method with a string. The parser will return the syntax tree of the match or `nil` if there is a failure.
13
13
 
14
14
  Treetop.load "arithmetic"
15
15
 
16
16
  parser = ArithmeticParser.new
17
- puts parser.parse('1+1').success?
17
+ if parser.parse('1+1')
18
+ puts 'success'
19
+ else
20
+ puts 'failure'
21
+ end
@@ -948,7 +948,7 @@ module Treetop
948
948
  prefix.compile(address, builder, self)
949
949
  end
950
950
 
951
- def predicated_expression
951
+ def prefixed_expression
952
952
  atomic
953
953
  end
954
954
 
@@ -1275,7 +1275,7 @@ module Treetop
1275
1275
  prefix.compile(lexical_address, builder, self)
1276
1276
  end
1277
1277
 
1278
- def predicated_expression
1278
+ def prefixed_expression
1279
1279
  elements[1]
1280
1280
  end
1281
1281
 
@@ -110,7 +110,7 @@ module Treetop
110
110
  prefix.compile(address, builder, self)
111
111
  end
112
112
 
113
- def predicated_expression
113
+ def prefixed_expression
114
114
  atomic
115
115
  end
116
116
 
@@ -206,7 +206,7 @@ module Treetop
206
206
  prefix.compile(lexical_address, builder, self)
207
207
  end
208
208
 
209
- def predicated_expression
209
+ def prefixed_expression
210
210
  elements[1]
211
211
  end
212
212
 
@@ -273,7 +273,7 @@ module Treetop
273
273
  end
274
274
 
275
275
  rule prefix
276
- '&' <AndPredicate> / '!' <NotPredicate>
276
+ '&' <AndPredicate> / '!' <NotPredicate> / '~' <TransientPrefix>
277
277
  end
278
278
 
279
279
  rule atomic
@@ -16,3 +16,4 @@ require File.join(dir, *%w[node_classes choice])
16
16
  require File.join(dir, *%w[node_classes repetition])
17
17
  require File.join(dir, *%w[node_classes optional])
18
18
  require File.join(dir, *%w[node_classes predicate])
19
+ require File.join(dir, *%w[node_classes transient_prefix])
@@ -4,7 +4,7 @@ module Treetop
4
4
  def compile(address, builder, parent_expression = nil)
5
5
  super
6
6
 
7
- builder.if__ "input.index(/#{text_value}/, index) == index" do
7
+ builder.if__ "input.index(/#{escaped_text_value}/, index) == index" do
8
8
  assign_result "(#{node_class_name}).new(input, index...(index + 1))"
9
9
  extend_result_with_inline_module
10
10
  builder << "@index += 1"
@@ -14,6 +14,10 @@ module Treetop
14
14
  assign_result 'nil'
15
15
  end
16
16
  end
17
+
18
+ def escaped_text_value
19
+ text_value.gsub(/\/|#(@|\$)/) {|match| "\\#{match}"}
20
+ end
17
21
  end
18
22
  end
19
23
  end
@@ -6,7 +6,7 @@ module Treetop
6
6
  begin_comment(parent_expression)
7
7
  use_vars :result, :start_index
8
8
  obtain_new_subexpression_address
9
- parent_expression.predicated_expression.compile(subexpression_address, builder)
9
+ parent_expression.prefixed_expression.compile(subexpression_address, builder)
10
10
  builder.if__(subexpression_success?) { when_success }
11
11
  builder.else_ { when_failure }
12
12
  end_comment(parent_expression)
@@ -0,0 +1,9 @@
1
+ module Treetop
2
+ module Compiler
3
+ class TransientPrefix < ParsingExpression
4
+ def compile(address, builder, parent_expression)
5
+ parent_expression.prefixed_expression.compile(address, builder)
6
+ end
7
+ end
8
+ end
9
+ end
@@ -3,4 +3,5 @@ require "#{dir}/runtime/compiled_parser"
3
3
  require "#{dir}/runtime/syntax_node"
4
4
  require "#{dir}/runtime/node_cache"
5
5
  require "#{dir}/runtime/parse_cache"
6
- require "#{dir}/runtime/terminal_parse_failure"
6
+ require "#{dir}/runtime/terminal_parse_failure"
7
+ require "#{dir}/runtime/interval_skip_list"
@@ -0,0 +1,4 @@
1
+ dir = File.dirname(__FILE__)
2
+ require "#{dir}/interval_skip_list/interval_skip_list.rb"
3
+ require "#{dir}/interval_skip_list/head_node.rb"
4
+ require "#{dir}/interval_skip_list/node.rb"
@@ -0,0 +1,15 @@
1
+ class IntervalSkipList
2
+ class HeadNode
3
+ attr_reader :height, :forward, :forward_markers
4
+
5
+ def initialize(height)
6
+ @height = height
7
+ @forward = Array.new(height, nil)
8
+ @forward_markers = Array.new(height) {|i| []}
9
+ end
10
+
11
+ def top_level
12
+ height - 1
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,200 @@
1
+ class IntervalSkipList
2
+ attr_reader :probability
3
+
4
+ def initialize
5
+ @head = HeadNode.new(max_height)
6
+ @ranges = {}
7
+ @probability = 0.5
8
+ end
9
+
10
+ def max_height
11
+ 3
12
+ end
13
+
14
+ def empty?
15
+ head.forward[0].nil?
16
+ end
17
+
18
+ def expire(range, length_change)
19
+ expired_markers, first_node_after_range = overlapping(range)
20
+ expired_markers.each { |marker| delete(marker) }
21
+ first_node_after_range.propagate_length_change(length_change)
22
+ end
23
+
24
+ def overlapping(range)
25
+ markers, first_node = containing_with_node(range.first)
26
+
27
+ cur_node = first_node
28
+ begin
29
+ markers.concat(cur_node.forward_markers.flatten)
30
+ cur_node = cur_node.forward[0]
31
+ end while cur_node.key < range.last
32
+
33
+ return markers.uniq, cur_node
34
+ end
35
+
36
+ def containing(n)
37
+ containing_with_node(n).first
38
+ end
39
+
40
+ def insert(range, marker)
41
+ ranges[marker] = range
42
+ first_node = insert_node(range.first)
43
+ first_node.endpoint_of.push(marker)
44
+ last_node = insert_node(range.last)
45
+ last_node.endpoint_of.push(marker)
46
+
47
+ cur_node = first_node
48
+ cur_level = first_node.top_level
49
+ while next_node_at_level_inside_range?(cur_node, cur_level, range)
50
+ while can_ascend_from?(cur_node, cur_level) && next_node_at_level_inside_range?(cur_node, cur_level + 1, range)
51
+ cur_level += 1
52
+ end
53
+ cur_node = mark_forward_path_at_level(cur_node, cur_level, marker)
54
+ end
55
+
56
+ while node_inside_range?(cur_node, range)
57
+ while can_descend_from?(cur_level) && next_node_at_level_outside_range?(cur_node, cur_level, range)
58
+ cur_level -= 1
59
+ end
60
+ cur_node = mark_forward_path_at_level(cur_node, cur_level, marker)
61
+ end
62
+ end
63
+
64
+ def delete(marker)
65
+ range = ranges[marker]
66
+ path_to_first_node = make_path
67
+ first_node = find(range.first, path_to_first_node)
68
+
69
+ cur_node = first_node
70
+ cur_level = first_node.top_level
71
+ while next_node_at_level_inside_range?(cur_node, cur_level, range)
72
+ while can_ascend_from?(cur_node, cur_level) && next_node_at_level_inside_range?(cur_node, cur_level + 1, range)
73
+ cur_level += 1
74
+ end
75
+ cur_node = unmark_forward_path_at_level(cur_node, cur_level, marker)
76
+ end
77
+
78
+ while node_inside_range?(cur_node, range)
79
+ while can_descend_from?(cur_level) && next_node_at_level_outside_range?(cur_node, cur_level, range)
80
+ cur_level -= 1
81
+ end
82
+ cur_node = unmark_forward_path_at_level(cur_node, cur_level, marker)
83
+ end
84
+ last_node = cur_node
85
+
86
+ first_node.endpoint_of.delete(marker)
87
+ if first_node.endpoint_of.empty?
88
+ first_node.delete(path_to_first_node)
89
+ end
90
+
91
+ last_node.endpoint_of.delete(marker)
92
+ if last_node.endpoint_of.empty?
93
+ path_to_last_node = make_path
94
+ find(range.last, path_to_last_node)
95
+ last_node.delete(path_to_last_node)
96
+ end
97
+ end
98
+
99
+ protected
100
+ attr_reader :head, :ranges
101
+
102
+ def insert_node(key)
103
+ path = make_path
104
+ found_node = find(key, path)
105
+ if found_node && found_node.key == key
106
+ return found_node
107
+ else
108
+ return Node.new(key, next_node_height, path)
109
+ end
110
+ end
111
+
112
+ def containing_with_node(n)
113
+ containing = []
114
+ cur_node = head
115
+ (max_height - 1).downto(0) do |cur_level|
116
+ while (next_node = cur_node.forward[cur_level]) && next_node.key <= n
117
+ cur_node = next_node
118
+ if cur_node.key == n
119
+ return containing + (cur_node.markers - cur_node.endpoint_of), cur_node
120
+ end
121
+ end
122
+ containing.concat(cur_node.forward_markers[cur_level])
123
+ end
124
+
125
+ return containing, cur_node
126
+ end
127
+
128
+ def delete_node(key)
129
+ path = make_path
130
+ found_node = find(key, path)
131
+ found_node.delete(path) if found_node.key == key
132
+ end
133
+
134
+ def find(key, path)
135
+ cur_node = head
136
+ (max_height - 1).downto(0) do |cur_level|
137
+ while (next_node = cur_node.forward[cur_level]) && next_node.key < key
138
+ cur_node = next_node
139
+ end
140
+ path[cur_level] = cur_node
141
+ end
142
+ cur_node.forward[0]
143
+ end
144
+
145
+ def make_path
146
+ Array.new(max_height, nil)
147
+ end
148
+
149
+ def next_node_height
150
+ height = 1
151
+ while rand < probability && height < max_height
152
+ height += 1
153
+ end
154
+ height
155
+ end
156
+
157
+ def can_ascend_from?(node, level)
158
+ level < node.top_level
159
+ end
160
+
161
+ def can_descend_from?(level)
162
+ level > 0
163
+ end
164
+
165
+ def node_inside_range?(node, range)
166
+ node.key < range.last
167
+ end
168
+
169
+ def next_node_at_level_inside_range?(node, level, range)
170
+ node.forward[level] && node.forward[level].key <= range.last
171
+ end
172
+
173
+ def next_node_at_level_outside_range?(node, level, range)
174
+ (node.forward[level].nil? || node.forward[level].key > range.last)
175
+ end
176
+
177
+ def mark_forward_path_at_level(node, level, marker)
178
+ node.forward_markers[level].push(marker)
179
+ next_node = node.forward[level]
180
+ next_node.markers.push(marker)
181
+ node = next_node
182
+ end
183
+
184
+ def unmark_forward_path_at_level(node, level, marker)
185
+ node.forward_markers[level].delete(marker)
186
+ next_node = node.forward[level]
187
+ next_node.markers.delete(marker)
188
+ node = next_node
189
+ end
190
+
191
+ def nodes
192
+ nodes = []
193
+ cur_node = head.forward[0]
194
+ until cur_node.nil?
195
+ nodes << cur_node
196
+ cur_node = cur_node.forward[0]
197
+ end
198
+ nodes
199
+ end
200
+ end