treetop 1.3.0 → 1.4.1

data/bin/tt

@@ -1,28 +1,112 @@
 #!/usr/bin/env ruby
+require 'optparse'
 require 'rubygems'
 gem 'treetop'
 
 $LOAD_PATH.unshift(File.expand_path(File.dirname(__FILE__) + "/../lib"))
 require 'treetop'
+require 'treetop/version'
+
+options = {}
+parser = OptionParser.new do |opts|
+  exts = Treetop::VALID_GRAMMAR_EXT.collect { |i| '.' + i }
+
+  opts.banner = "Treetop Parsing Expression Grammar (PEG) Comand Line Compiler"
+  opts.define_head "Usage: tt [options] grammar_file[#{exts.join('|')}] ..."
+  opts.separator ''
+  opts.separator 'Examples:'
+  opts.separator '  tt foo.tt               # 1 grammar -> 1 parser source'
+  opts.separator '  tt foo bar.treetop      # 2 grammars -> 2 separate parsers'
+  opts.separator '  tt -o alt_name.rb foo   # alternately named output file'
+  opts.separator ''
+  opts.separator ''
+  opts.separator 'NOTE: while treetop grammar files *must* have one of the following'
+  opts.separator 'filename extensions, the extension name is not required when calling'
+  opts.separator 'the compiler with grammar file names.'
+  opts.separator ''
+  opts.separator "  Valid extensions: #{exts.join(', ')}"
+  opts.separator ''
+  opts.separator ''
+  opts.separator 'Options:'
+
+  opts.on('-o', '--output FILENAME', 'Write parser source to FILENAME') do |fn|
+    options[:out_file] = fn
+  end
+
+  opts.on('-f', '--force', 'Overwrite existing output file(s)') do
+    options[:force] = true
+  end
+
+  opts.on_tail('-v', '--version', 'Show Treetop version') do
+    puts "Treetop v#{Treetop::VERSION::STRING}"
+    exit
+  end
+
+  opts.on_tail('-h', '--help', 'Show this help message') do
+    puts opts
+    exit
+  end
 
-if ARGV.empty?
-  puts "Usage:\n\ntt foo.treetop bar.treetop ...\n or\ntt foo.treetop -o alternate_name.rb\n\n"
-  exit
+end
+file_list = parser.parse!
+
+# check options and arg constraints
+if file_list.empty? || (options[:out_file] && file_list.size > 1)
+  puts parser
+  exit 1
+end
+
+def grammar_exist?(filename)
+  if File.extname(filename).empty?
+    Treetop::VALID_GRAMMAR_EXT.each do |ext|
+      fn_ext = "#{filename}.#{ext}"
+      return true if File.exist?(fn_ext) && !File.zero?(fn_ext)
+    end
+  end
+  File.exist?(filename) && !File.zero?(filename)
+end
+
+def full_grammar_filename(filename)
+  return filename if !File.extname(filename).empty?
+  Treetop::VALID_GRAMMAR_EXT.each do |ext|
+    fn_ext = "#{filename}.#{ext}"
+    return fn_ext if File.exist?(fn_ext) && !File.zero?(fn_ext)
+  end
+end
+
+def protect_output?(filename, forced=false)
+  if !forced and
+      File.exist?(filename) and
+      (l=File.open(filename) { |f| f.gets rescue "" }) != Treetop::Compiler::AUTOGENERATED
+    puts "ERROR: '#{filename}' output already exists; skipping compilation...\n"
+    return true
+  end
+  false
 end
 
 compiler = Treetop::Compiler::GrammarCompiler.new
 
-while !ARGV.empty?
-  treetop_file = ARGV.shift
-  if !File.exist?(treetop_file)
-    puts "Error: file '#{treetop_file}' doesn't exist\n\n"
-    exit(2) 
+while !file_list.empty?
+  treetop_file = file_list.shift
+
+  # handle nonexistent and existent grammar files mixed together
+  if !grammar_exist?(treetop_file)
+    puts "ERROR: input grammar file '#{treetop_file}' does not exist; continuing...\n"
+    next
   end
-  if ARGV.size >= 2 && ARGV[1] == '-o'
-    ARGV.shift	  # explicit output file name option
-    compiler.compile(treetop_file, ARGV.shift)
+
+  # try to compile
+  treetop_file = full_grammar_filename(treetop_file)
+  std_output_file = treetop_file.gsub(Treetop::VALID_GRAMMAR_EXT_REGEXP, '.rb')
+
+  if options[:out_file]
+    # explicit output file name option; never overwrite unless forced
+    next if protect_output?(options[:out_file], options[:force])
+    compiler.compile(treetop_file, options[:out_file])
   else
-    # list of input files option
+    # compile one input file from input file list option; never overwrite unless forced
+    next if protect_output?(std_output_file, options[:force])
     compiler.compile(treetop_file)
   end
+
 end

data/doc/semantic_interpretation.markdown

@@ -45,7 +45,7 @@ Methods can be added to the nodes instantiated by the successful match of an exp
       end
     end
 
-Note that each alternative expression is followed by a block containing a method definition. A `depth` method is defined on both expressions. The recursive `depth` method defined in the block following the first expression determines the depth of the nested parentheses and adds one two it. The base case is implemented in the block following the second expression; a single character has a depth of 0.
+Note that each alternative expression is followed by a block containing a method definition. A `depth` method is defined on both expressions. The recursive `depth` method defined in the block following the first expression determines the depth of the nested parentheses and adds one to it. The base case is implemented in the block following the second expression; a single character has a depth of 0.
 
 
 ###Custom `SyntaxNode` Subclass Declarations
@@ -186,4 +186,4 @@ The module containing automatically defined element accessor methods is an ances
       Available only on nonterminal nodes, returns the nodes parsed by the elements of the matched sequence.
     </td>
   </tr>
-</table>
+</table>

data/doc/site.rb

@@ -1,6 +1,8 @@
 require 'rubygems'
 require 'erector'
 require "#{File.dirname(__FILE__)}/sitegen"
+require 'fileutils'
+require 'bluecloth'
 
 class Layout < Erector::Widget
   def render

data/doc/sitegen.rb

@@ -7,6 +7,7 @@ class Layout < Erector::Widget
     end
 
     def generate_site
+      FileUtils.mkdir_p(site_dir)
       @@page_classes.each do |page_class|
         page_class.generate_html unless page_class.abstract?
         puts page_class
@@ -28,7 +29,7 @@ class Layout < Erector::Widget
     end
 
     def absolutize(relative_path)
-      File.join(File.dirname(__FILE__), "site", relative_path)
+      File.join(site_dir, relative_path)
     end
 
     def abstract
@@ -38,6 +39,10 @@ class Layout < Erector::Widget
     def abstract?
       @abstract
     end
+
+    def site_dir
+      File.join(File.dirname(__FILE__), "site")
+    end
   end
 
   def bluecloth(relative_path)

data/doc/using_in_ruby.markdown

@@ -19,3 +19,129 @@ If a grammar by the name of `Foo` is defined, the compiled Ruby source will defi
     else
       puts 'failure'
     end
+
+##Defining Grammars Directly in Ruby
+It is possible to define parser directly in Ruby source file.
+
+###Grammars
+Defining parsers in Ruby code is as much similar to original definition as it is possible. To create a grammar just write:
+
+    include Treetop::Syntax
+    grammar :Foo do
+    end
+    parser = FooParser.new
+
+Treetop will automatically compile and load it into memory, thus an instance of `FooParser` can be created.
+
+###Syntactic Recognition
+To create a rule inside of a grammar simply write:
+
+    include Treetop::Syntax
+    grammar :Foo do
+      rule :bar do
+        ...
+      end
+    end
+
+Inside the rule any of Treetop syntactic elements can be used. Each element of a rule is created with standard Ruby classes: Strings act as Terminals, Symbols stand for Nonterminals, Arrays are sequences, Regexps are character classes.
+
+_Note: it is better not to use Numbers, as terminal symbols; use Strings instead._
+
+Sequences can be defined as follows:
+
+    rule :sequence do
+      [ "foo", "bar", "baz" ]
+    end
+
+Ordered choices use `/` operator:
+
+    rule :choice do
+      "foo" / "bar"
+    end
+
+Sequences have higher precedence than choices, so choices must be parenthesized to be used as the elements of a sequence. For example:
+
+    rule :nested do
+      ["foo", "bar", "baz" / "bop" ]    # -> "foo" "bar" ( "baz" / "bop" )
+    end
+
+Special operators like `!`, `&`, `?`, `+` and `*` are available through methods (all of the methods return element itself so calls can be chained) of elements in a rule:
+
+    Op. | Method
+    -----------
+     !  | bang
+     &  | amper
+     ?  | mark
+     +  | plus
+     *  | kleene
+
+For example grammar:
+
+    grammar :Foo do
+      rule :bar do
+       [ "baz" / "bop" ].kleene
+      end
+    end
+
+can generate any word that contain words "bar" and "bop".
+
+###Semantic Interpretation
+
+Syntax node declaration can be added by `node` method (which may be called the same as operators above):
+
+    grammar :Parens do
+      rule :parenthesized_letter do
+        ([ '(', :parenthesized_letter, ')'] / /[a-z]/ ).node(:ParenNode)
+      end
+    end
+    
+It is also possible to add inline blocks of code. They are in fact strings strictly inserted into generated grammar:
+
+    grammar :Parens do
+      rule :parenthesized_letter do
+        (['(', :parenthesized_letter, ')'] / /[a-z]/ ).block(%{
+          def depth
+            if nonterminal?
+              parenthesized_letter.depth + 1
+            else
+              0
+            end
+          end
+        })
+      end
+    end
+
+Labels in rule definitions can be written as follow (example taken from documentation):
+
+  rule :labels do
+    [/[a-z]/.label(:first_letter), [', ', /[a-z]/.kleene.label(:letter)].label(:rest_letters)].block(%{
+      ...
+    })
+  end
+
+###Composition
+
+Inclusion of a grammar works thanks to `include` function call inside the grammar definition:
+
+    grammar :One do
+      rule :a do
+        foo"
+      end
+    
+      rule :b do        
+        "baz"
+      end      
+    end
+
+    grammar :Two do
+      include :One
+      rule :a do      
+        :super / "bar" / :c
+      end
+      
+      rule :c do
+       :b      
+      end
+    end
+
+Grammar Two can generate `"foo"`, `"bar"` and `"baz"` words.

data/examples/ruby_syntax/syntax_test.rb

@@ -0,0 +1,105 @@
+dir = File.dirname(__FILE__)
+require File.expand_path("#{dir}/test_helper")
+
+class SyntaxTest < Test::Unit::TestCase
+	include Treetop::Syntax
+	include SyntaxTestHelper
+
+	def test_simple
+		assert_grammar {
+			grammar :OnlyGrammar do
+			end
+		}
+	end
+
+	def test_rules
+		assert_grammar {
+			grammar :Simple do
+				rule :foo do
+					["foo", :bar]
+				end
+
+				rule :bar do
+					"bar" / "baz"
+				end
+			end
+		}
+		parse('foobar')
+		parse('foobaz')
+	end
+
+	def test_nested
+		assert_grammar {
+			grammar :Nested do
+				rule :nested do
+					["foo", "bar", "baz" / "bop"]
+				end
+			end
+		}
+		parse('foobarbaz')
+		parse('foobarbop')
+	end
+
+	def test_operators
+		assert_grammar {
+			grammar :Kleene do
+				rule :Kleene do
+					"foo".kleene
+				end
+			end
+		}
+		parse("")
+		parse("foo")
+		parse("foofoo")
+
+		assert_grammar {
+			grammar :Plus do
+				rule :Plus do
+					"foo".plus
+				end
+			end
+		}
+		parse("foo")
+		parse("foofoo")
+
+		assert_grammar {
+			grammar :Optional do
+				rule :Optional do
+					"foo".mark
+				end
+			end
+		}
+		parse("")
+		parse("foo")
+	end
+
+	def test_inclusion
+		assert_grammar {
+			grammar :One do
+				rule :a do
+					"foo"
+				end
+
+				rule :b do
+					"baz"
+				end
+			end
+		}
+
+		assert_grammar {
+			grammar :Two do
+				include :One
+				rule :a do
+					:super / "bar" / :c
+				end
+
+				rule :c do
+					:b
+				end
+			end
+		}
+		parse("foo")
+		parse("bar")
+		parse("baz")
+	end
+end

data/examples/ruby_syntax/test_helper.rb

@@ -0,0 +1,28 @@
+require 'test/unit'
+require 'rubygems'
+require 'treetop'
+
+dir = File.dirname(__FILE__)
+require File.expand_path("#{dir}/../../lib/treetop/ruby_extensions")
+require File.expand_path("#{dir}/../../lib/treetop/syntax")
+
+module SyntaxTestHelper
+	def assert_grammar
+		g = yield
+		assert_not_nil g
+		flunk "Badly generated parser" unless g
+		@parser = eval("#{g}.new")
+	end
+
+	def parse(input)
+		result = @parser.parse(input)
+    unless result
+      puts @parser.terminal_failures.join("\n")
+    end
+    assert_not_nil result
+		if result
+			assert_equal input, result.text_value
+		end
+    result
+	end
+end

data/lib/treetop/compiler/grammar_compiler.rb

@@ -1,8 +1,10 @@
 module Treetop
   module Compiler
+    AUTOGENERATED = "# Autogenerated from a Treetop grammar. Edits may be lost.\n"
     class GrammarCompiler
       def compile(source_path, target_path = source_path.gsub(/\.(treetop|tt)\Z/, '.rb'))
         File.open(target_path, 'w') do |target_file|
+          target_file.write(AUTOGENERATED+"\n\n")
           target_file.write(ruby_source(source_path))
         end
       end