hipe-gorillagrammar 0.0.1beta

data/.gitignore

@@ -0,0 +1,21 @@
+coverage
+.yardoc
+.DS_Store
+pkg
+doc
+ri
+email.txt
+.svn
+log
+.project
+.loadpath
+*.swp
+results
+test_apps
+*.tmproj
+*.log
+*.pid
+bin
+vendor/gems
+eraseme.*
+tmp.*

data/History.txt

@@ -0,0 +1,8 @@
+== 0.0.1 / 2009-11-30
+* 100% C0 & C1 test coverage
+* under 500 LOC
+* I think this is an LR parser generator, but i'm not sure yet
+
+
+== 0.0.0 / 2009-11-24
+* birthday (but see the original in php on my old laptop)

data/LICENSE.txt

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Mark Meves
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.txt

@@ -0,0 +1,11 @@
+Experiments in parsing.   Previous version was in the "Githelper" script of mine on github.
+
+(The original version was a php class i wrote many years ago with the same name, ...@todo)
+
+ThankYou's
+ Brian Helkamp and the webrat team for giving me an excellent Thorfile and teach examples of rspec
+ argv[0]    Sun Nov 29 00:05:46 EST 2009 in #ruby-lang for schooling me on module inheritance chains
+ hagabaka   Mon Nov 30 02:15:51 EST 2009
+ raggi      for suggesting Marshal.dump/Marshal.load
+ 
+Above all, thank you Yoko for putting up w/ me

data/Rakefile

@@ -0,0 +1,25 @@
+
+# require 'spec'
+require 'spec/rake/spectask'
+require 'spec/rake/verify_rcov'
+
+desc "Run API and Core specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+  #t.spec_files = FileList['spec/public/**/*_spec.rb'] + FileList['spec/private/**/*_spec.rb']
+  t.spec_files = FileList['spec/**/*_spec.rb'] + FileList['spec/private/**/*_spec.rb']  
+end
+
+desc "Run all examples with RCov"
+Spec::Rake::SpecTask.new('rcov') do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.rcov = true
+  t.rcov_opts = ['--exclude', 'spec,/Library/Ruby/Gems/1.8/gems']
+end
+
+RCov::VerifyTask.new(:rcovv => 'rcov') do |t|
+  t.threshold = 95.0
+  t.index_html = 'coverage/index.html'
+end
+
+

data/Thorfile

@@ -0,0 +1,135 @@
+# this file was originally copy-pasted from webrat's Thorfile.  Thank you Bryan Helmkamp!
+require 'ruby-debug'
+
+module GemHelpers
+
+  def generate_gemspec
+    $LOAD_PATH.unshift(File.expand_path(File.join(File.dirname(__FILE__), "lib")))
+    require "hipe-gorillagrammar"
+    
+    Gem::Specification.new do |s|    
+      s.name      = 'hipe-gorillagrammar'
+      s.version   = Hipe::GorillaGrammar::VERSION
+      s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+      s.authors   = ["Mark Meves"]
+      s.email     = "mark.meves@gmail.com"
+      s.homepage  = "http://github.com/hipe/hipe-gorillagrammar"
+      s.date      = %q{2009-11-23}  
+      s.summary   = %q{'beta attempt at a simple LR parser generator driven by DSL under 500LOC 100% C1 test coverage'}
+      s.description  = <<-EOS.strip
+      LR Parser Generator (?) under 500 LOC with 100*% C1 test coverage.  No useful AST yet.  No useful docs yet.
+      EOS
+      
+      # s.rubyforge_project = "webrat"
+
+      require "git"
+      repo = Git.open(".")
+
+      s.files      = normalize_files(repo.ls_files.keys - repo.lib.ignored_files)
+      s.test_files = normalize_files(Dir['spec/***.rb'] - repo.lib.ignored_files)
+      #s.test_files = normalize_files(Dir['spec/*.rb'] - repo.lib.ignored_files)  
+
+      s.has_rdoc = 'yard'  # trying out arg[0]/lsegal's doc tool
+      #s.extra_rdoc_files = %w[README.rdoc MIT-LICENSE.txt History.txt]
+      #s.extra_rdoc_files = %w[MIT-LICENSE.txt History.txt]
+
+      #s.add_dependency "nokogiri", ">= 1.2.0"
+      #s.add_dependency "rack", ">= 1.0"
+    end
+  end
+
+  def normalize_files(array)
+    # only keep files, no directories, and sort
+    array.select do |path|
+      File.file?(path)
+    end.sort
+  end
+
+  # Adds extra space when outputting an array. This helps create better version
+  # control diffs, because otherwise it is all on the same line.
+  def prettyify_array(gemspec_ruby, array_name)
+    gemspec_ruby.gsub(/s\.#{array_name.to_s} = \[.+?\]/) do |match|
+      leadin, files = match[0..-2].split("[")
+      leadin + "[\n    #{files.split(",").join(",\n   ")}\n  ]"
+    end
+  end
+
+  def read_gemspec
+    @read_gemspec ||= eval(File.read("hipe-gorillagrammar.gemspec"))
+  end
+
+  def sh(command)
+    puts command
+    system command
+  end
+end
+
+class Default < Thor
+  include GemHelpers
+
+  desc "gemspec", "Regenerate hipe-gorillagrammar.gemspec"
+  def gemspec
+    File.open("hipe-gorillagrammar.gemspec", "w") do |file|
+      gemspec_ruby = generate_gemspec.to_ruby
+      gemspec_ruby = prettyify_array(gemspec_ruby, :files)
+      gemspec_ruby = prettyify_array(gemspec_ruby, :test_files)
+      gemspec_ruby = prettyify_array(gemspec_ruby, :extra_rdoc_files)
+
+      file.write gemspec_ruby
+    end
+
+    puts "Wrote gemspec to hipe-gorillagrammar.gemspec"
+    read_gemspec.validate
+  end
+
+  desc "build", "Build a hipe-gorillagrammar gem"
+  def build
+    sh "gem build hipe-gorillagrammar.gemspec"
+    FileUtils.mkdir_p "pkg"
+    FileUtils.mv read_gemspec.file_name, "pkg"
+  end
+
+  desc "install", "Install the latest built gem"
+  def install
+    sh "gem install --local pkg/#{read_gemspec.file_name}"
+  end
+
+  desc "release", "Release the current branch to GitHub and Gemcutter"
+  def release
+    gemspec
+    build
+    Release.new.tag
+    Release.new.gem
+  end
+  
+end
+
+
+class Spec < Thor
+  desc "spec", "el speco"
+  def run which='all'
+    t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
+    t.spec_files = FileList['spec/**/*_spec.rb'] + FileList['spec/**/*_spec.rb']
+    debugger
+    'x'
+    
+  end
+end
+
+
+
+class Release < Thor
+  include GemHelpers
+
+  desc "tag", "Tag the gem on the origin server"
+  def tag
+    release_tag = "v#{read_gemspec.version}"
+    sh "git tag -a #{release_tag} -m 'Tagging #{release_tag}'"
+    sh "git push origin #{release_tag}"
+  end
+
+  desc "gem", "Push the gem to Gemcutter"
+  def gem
+    sh "gem push pkg/#{read_gemspec.file_name}"
+  end
+end

data/hipe-gorillagrammar.gemspec

@@ -0,0 +1,66 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{hipe-gorillagrammar}
+  s.version = "0.0.1beta"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Mark Meves"]
+  s.date = %q{2009-11-23}
+  s.description = %q{LR Parser Generator (?) under 500 LOC with 100*% C1 test coverage.  No useful AST yet.  No useful docs yet.}
+  s.email = %q{mark.meves@gmail.com}
+  s.files = [
+    ".gitignore",
+    "History.txt",
+    "LICENSE.txt",
+    "README.txt",
+    "Rakefile",
+    "Thorfile",
+    "hipe-gorillagrammar.gemspec",
+    "lib/hipe-gorillagrammar.rb",
+    "lib/hipe-gorillagrammar/extensions/syntax.rb",
+    "spec/argv.rb",
+    "spec/extensions/syntax_spec.rb",
+    "spec/grammar_spec.rb",
+    "spec/helpers.rb",
+    "spec/parse_tree_spec.rb",
+    "spec/parsing_spec.rb",
+    "spec/range_spec.rb",
+    "spec/regexp_spec.rb",
+    "spec/runtime_spec.rb",
+    "spec/sequence_spec.rb",
+    "spec/shorthand_spec.rb",
+    "spec/spec.opts",
+    "spec/symbol_reference_spec.rb",
+    "spec/symbol_spec.rb"
+  ]
+  s.has_rdoc = %q{yard}
+  s.homepage = %q{http://github.com/hipe/hipe-gorillagrammar}
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{'beta attempt at a simple LR parser generator driven by DSL under 500LOC 100% C1 test coverage'}
+  s.test_files = [
+    "spec/argv.rb",
+    "spec/grammar_spec.rb",
+    "spec/helpers.rb",
+    "spec/parse_tree_spec.rb",
+    "spec/parsing_spec.rb",
+    "spec/range_spec.rb",
+    "spec/regexp_spec.rb",
+    "spec/runtime_spec.rb",
+    "spec/sequence_spec.rb",
+    "spec/shorthand_spec.rb",
+    "spec/symbol_reference_spec.rb",
+    "spec/symbol_spec.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/lib/hipe-gorillagrammar/extensions/syntax.rb

@@ -0,0 +1,50 @@
+module Hipe::GorillaGrammar
+  module GorillaSymbol
+    def syntax_pretty_name; @name ? @name.to_s.upcase : nil end
+  end
+  
+  module RegexpTerminal
+    def syntax_tokens; [syntax] end
+    def syntax; syntax_pretty_name || self end
+  end
+
+  module StringTerminal
+    def syntax_tokens; [syntax] end
+    def syntax; self end
+  end
+  
+  class SymbolReference
+    def syntax_tokens; [syntax_pretty_name] end
+  end
+  
+  module NonTerminalSymbol
+    def syntax
+      return (syntax_tokens * ' ').gsub(%r{(?:[\[\(] +| +[\]\)]| +\| +)}){|x| x.strip}
+    end
+  end
+
+  class Sequence
+    def syntax_tokens
+      @group.map{ |x| x.syntax_tokens }.flatten
+    end
+  end
+  
+  class RangeOf
+    def syntax_tokens;
+      is_one_or_more = (@range == (1..Infinity) and @group.size == 1 and @group[0].kind_of? SymbolReference)
+      things = case true
+        when (@range == (0..1))        then ['[', nil, ']']
+        when (@range == (1..1))        then ['(', nil, ')']
+        when (@range == (0..Infinity)) then ['[', '[...]', ']']
+        when (is_one_or_more) then ['[', '[...]', ']'] # *special handing below
+        else [%{(#{@range.to_s}) of (}, nil, ')']
+      end
+      thing = @group.map{ |x| x.syntax_tokens }.zip(Array.new(@group.size-1,'|')).flatten.compact
+      thing.unshift things[0]
+      thing.push things[1] if things[1]
+      thing.push things[2]
+      thing.unshift @group[0].syntax_pretty_name if is_one_or_more
+      thing
+    end
+  end      
+end

data/lib/hipe-gorillagrammar.rb

@@ -0,0 +1,492 @@
+require 'rubygems'
+#require 'ruby-debug'
+require 'singleton'
+
+class Symbol 
+  def satisfied?; @satisfied; end
+  def accepting?; @accepting; end
+end
+# these are the different states that a parsing node can have
+# after it's consumed a token. absurd hack so we can use smiley faces as symbols  (note 2)
+# mouth open means "still accepting".  Smiley means "is satisfied".  four permutations
+:D.instance_variable_set :@satisfied, true  # open mouth happy
+:D.instance_variable_set :@accepting, true 
+:>.instance_variable_set :@satisfied, true  # closed mouth happy
+:>.instance_variable_set :@accepting, false 
+:O.instance_variable_set :@satisfied, false # open mouth unhappy
+:O.instance_variable_set :@accepting, true 
+:C.instance_variable_set :@satisfied, false # closed mouth unhappy
+:C.instance_variable_set :@accepting, false 
+module Hipe
+  def self.GorillaGrammar opts=nil, &block
+    GorillaGrammar.define(opts, &block)
+  end
+  module GorillaGrammar
+    VERSION = '0.0.1beta'
+    Infinity = 1.0 / 0
+    def self.define(opts=nil, &block)
+      runtime = Runtime.instance
+      g = runtime.create_grammar! opts 
+      runtime.push_grammar g 
+      begin
+        g.define(&block)
+      ensure
+        runtime.pop_grammar 
+      end
+    end
+    class Runtime # for global-like stuff
+      include Singleton
+      def initialize 
+        @grammar_stack = []
+        @grammars = {}
+      end
+      def push_grammar(grammar)
+        raise UsageFailure.new('sorry, no making grammars inside of grammars') if @grammar_stack.size > 0
+        @grammar_stack << grammar
+      end
+      def pop_grammar; @grammar_stack.pop; end
+      def current_grammar!
+        raise UsageFailure.new("no current grammar") unless current_grammar
+        current_grammar 
+      end
+      def current_grammar;  @grammar_stack.last;  end
+      def self.method_missing(a,*b)
+        return instance.send(a,*b) if instance.respond_to? a
+        raise NoMethodError.new %{undefined method `#{a}' for #{inspect}}
+      end
+      def create_grammar! opts
+        opts ||= {}
+        g = Grammar.new opts 
+        g.name = %{grammar#{@grammars.size+1}} unless g.name
+        raise GorillaException.new("for now we can't reopen grammars") if @grammars[g.name]
+        @grammars[g.name] = g
+      end
+      def get_grammar name; @grammars[name]; end
+      # enables the use of  =~,  |,  (0..1).of .., .., .., and :some_name[/regexp/] in grammars
+      def enable_operator_shorthands
+        return if @shorthands_enabled
+        Symbol.instance_eval { include SymbolHack, PipeHack }
+        String.instance_eval { include PipeHack }
+        Fixnum.instance_eval { include FixnumOfHack }
+        Range.instance_eval { include RangeOfHack }
+        @shorthands_enabled = true 
+      end
+    end
+    class Grammar < Hash
+      attr_accessor :name, :root
+      alias_method :names, :keys
+      def initialize opts 
+        opts ||= {}
+        @name = opts[:name] if opts[:name]
+        @with_operator_shorthands = opts.has_key?(:enable_operator_shorthands) ? 
+          opts[:enable_operator_shorthands] : true
+      end
+      def == other; self.inspect == other.inspect end #hack      
+      def define(&block) # should it return grammar or last symbol? grammar. (note 4:)
+        Runtime.enable_operator_shorthands if @with_operator_shorthands
+        @root = GorillaSymbol.factory instance_eval(&block) # allows anonymous ranges & sequences as grammr
+        @root = @root.dereference if @root.instance_of? SymbolReference
+        self[@root.name = '__main__'] = @root if (@root.name.nil?)
+        self
+      end
+      def parse tox; @root.parse tox; end
+      def self.register_shorthand name, klass
+        instance_eval { 
+          define_method(name) { |*args|
+            klass.construct_from_shorthand(name, *args)
+          }
+        }
+      end
+      def []= name, symbol
+        raise GrammarGrammarException.new(%{Can't redefine symbols. Multiple definitions for :#{name}}) if self[name]
+        unless symbol.kind_of? GorillaSymbol
+          raise GorillaException.new(%{Expecting GorillaSymbol had #{symbol.inspect}}) 
+        end
+        symbol.name = name
+        super name, symbol
+      end
+    end
+    class GorillaException < Exception
+      def initialize(*args)
+        super args.shift if args[0].instance_of? String 
+        @info = args.last.instance_of?(Hash) ? args.pop : {}
+        @extra = args.count > 0 ? args : []
+      end
+      def tree; @info[:tree]; end
+    end
+    class UsageFailure < GorillaException; end
+    class GrammarGrammarException < UsageFailure; end 
+    class AmbiguousGrammar < GrammarGrammarException; end
+    class ParseFailure < GorillaException
+      def is_error?; true; end
+      def inspect; message; end # just for irb debugo 
+    end
+    class UnexpectedEndOfInput < ParseFailure; 
+      def message; <<-EOS.gsub(/^        /,'').gsub("\n",' ')
+        sorry, unexpected end of input.  I was
+        expecting you to say #{RangeOf.join(tree.expecting.uniq,', ',' or ')}.
+      EOS
+      end
+    end
+    class UnexpectedInput < ParseFailure;
+      def message
+        %{sorry, i don't know what you mean by "#{@info[:token]}".  } + 
+        ((ex = tree.expecting.uniq).size == 0 ? %{i wasn't expecting any more input.} :
+        %{i was expecting you to say #{RangeOf.join(ex,', ',' or ')}.})
+      end
+    end
+    module PipeHack
+      def |(other)
+        if self.instance_of? RangeOf
+          raise UsageFailure("no") unless self.is_pipe_hack
+          self << other
+          ret = self
+        else 
+          ret = RangeOf.new((1..1),[self,other])
+          ret.is_pipe_hack = true
+        end
+        ret
+      end
+    end
+    module FixnumOfHack # note that IRL, this will rarely be used for other than 1
+      def of *args
+        return RangeOf.new((1..1), args)
+      end
+    end
+    module RangeOfHack
+      def of *args
+        return RangeOf.new(self, args)
+      end
+    end
+    module SymbolHack
+      def =~ (symbol_data)
+        return super unless ( grammar = Runtime.instance.current_grammar )
+        grammar[self] = GorillaSymbol.factory symbol_data
+      end
+      def [] (*symbol_data)
+        return super unless Runtime.instance.current_grammar 
+        symbol_data = symbol_data[0] if symbol_data.size == 1
+        new_symbol = self.=~(symbol_data)
+        return SymbolReference.new self
+      end
+    end
+    module ParseTree; def is_error?; false end end
+    module GorillaSymbol # @abstract base
+      def self.factory obj # maps data structures to symbols. returns same object or new
+        case obj
+          when GorillaSymbol then obj # this one must stay on top!
+          when Array         then Sequence.new(*obj) 
+                             # note RangeOf is never constructed directly with factory()
+          when Symbol        then SymbolReference.new obj
+          when String        then obj.extend StringTerminal
+          when Regexp        then obj.extend RegexpTerminal
+          else
+            raise UsageFailure.new %{Can't determine symbol type for "#{obj.inspect}"},:obj=>obj
+        end 
+      end
+      def finalize; self; end
+      attr_accessor :name
+      attr_reader :kleene
+      def natural_name; name ? name.to_s.gsub('_',' ') : nil; end
+      def fork_for_parse; Marshal.load Marshal.dump(self); end # note 1
+      def reinit_for_parse; extend ParseTree end      
+      def prune! names=[]; a=(names&instance_variables); a.each{ |x| remove_instance_variable x }; a.size end
+      def dereference; self; end
+    end
+    module TerminalSymbol;
+      include GorillaSymbol;
+      attr_reader :status
+      def inspect; [@name ? @name.inspect : nil, super, %{(#{@token ? '.' : '_' })}].compact.join; end
+      def pretty_print q
+        q.text inspect
+        q.group 1,'{','}' do         
+          instance_variables.sort.each do |n|
+            next if instance_variable_get(n).nil?
+            q.text %{#{n}=}; q.pp instance_variable_get(n); q.text(';'); q.breakable 
+          end
+        end
+      end
+      def recurse &block; yield self end
+      def tokens; [@token]; end
+    end
+    module NonTerminalSymbol; 
+      include GorillaSymbol;
+      def [] (i); super(i) ? super(i) : @group[i]; end
+      def size; kind_of?(ParseTree) ? super : @group.size; end 
+      def == other
+        kind_of?(ParseTree) ? builtin_equality(other) : 
+        (other.class == self.class && @name == @name and @group == other.group)
+      end
+      attr_reader :group
+      def _inspect left, right, name = nil
+        name = %{:#{@name}} if name.nil? and @name
+        [name, left, 
+          (0..(@group ? @group.size : self.size)-1).map{|i| (self[i] ? self[i] : @group[i]).inspect}.join(', '),
+        right ].compact.join
+      end
+      def pretty_print q
+        names = instance_variables.sort{|a,b| (a=='@group') ? -1 : (b=='@group' ? 1 : a<=>b)}
+        names.delete_if{|x| instance_variable_get(x).nil?} # it may have been pruned
+        q.group 1,'{','}' do 
+          q.breakable
+          # show any non-nil properties of this object
+          names.each do |name|
+            q.text %{#{name}:}
+            q.pp instance_variable_get(name)
+            q.breakable
+          end
+          # show any captured children of this object (if it's parsed something)
+          each_with_index do |child, i|
+            q.text %{#{i}=>}
+            q.pp child
+            q.breakable
+          end 
+        end
+      end
+      def status; raise 'no' unless @status; @status end      
+      def _status=(smiley); raise 'no' unless smiley.kind_of? Symbol; @status = smiley end
+      def recurse &block
+        sum_like = yield self
+        self.each{ |x| if x.respond_to?(:recurse) then sum_like += x.recurse(&block) end }
+        sum_like
+      end
+      def tokens; [] end # for use in a call to recurse e.g. non_terminal.recurse{|x| x.tokens}
+    end
+    module StringTerminal # this could also be considered a special case of regexp terminal
+      include TerminalSymbol
+      def match token
+        status = if (self==token) 
+          @token = token
+          (:>)          
+        else
+          (:C)
+        end
+        @status = status
+      end
+      def expecting; [%{"#{self}"}]; end
+    end
+    module RegexpTerminal
+      include TerminalSymbol
+      Grammar.register_shorthand :regexp, self
+      def self.construct_from_shorthand name, *args
+        args[0].extend self
+      end
+      def [](capture_offset); @captures[capture_offset]; end
+      def expecting; @name ? [natural_name] : [self.inspect]; end 
+      def match token # cleaned up for note 5 @ 11/27 11:39 am
+        @status = if (md = super(token)) 
+          @captures = md.captures if (md.captures.size>0)
+          @token = token
+          (:>)
+        else
+          (:C)
+        end
+      end
+    end    
+    class SymbolReference
+      include GorillaSymbol
+      def inspect; %{::#{@name}}; end
+      def pp q; q.text(inspect); end
+      def initialize symbol
+        @name = symbol 
+        @grammar_name = Runtime.current_grammar!.name 
+      end
+      def dereference; 
+        @actual ||= Runtime.instance.get_grammar(@grammar_name)[@name].fork_for_parse.reinit_for_parse 
+      end 
+      def dereference_light
+        Runtime.instance.get_grammar(@grammar_name)[@name]        
+      end
+      [:kleene, :expecting, :reinit_for_parse].each do |name|
+        define_method(name){ dereference_light.send(name) }
+      end      
+    end
+    module CanParse
+      def parse tokens
+        tree = self.fork_for_parse.reinit_for_parse
+        while token = tokens.shift and tree.match(token).accepting?; end
+        if (:C) == tree.status or (tokens.size > 0) # xtra
+          UnexpectedInput.new :token=>(:C==tree.status ? token : tokens.first), :tree=>tree
+        elsif ! tree.status.satisfied?
+          UnexpectedEndOfInput.new :tree=>tree
+        else          
+          tree.finalize; #tree.prune!
+        end
+      end
+    end
+    class Sequence < Array
+      alias_method :builtin_equality, :==
+      include CanParse, NonTerminalSymbol, PipeHack
+      Grammar.register_shorthand :sequence, self
+      def self.construct_from_shorthand(name, *args); self.new(*args); end 
+      def initialize *args
+        @index = 0 # we use this to report expecting whether or not we are a parse
+        raise GrammarGrammarException.new "Arguments must be non-zero length" unless args.size > 0
+        @group = args.map{|x| GorillaSymbol.factory x }
+      end      
+      def reinit_for_parse;
+        super
+        @stop_here = @group.size;
+        num = @group.reverse.map{|x| 
+          x.reinit_for_parse
+          x.kleene 
+        }.find_index{|x| x==false || x.nil? } || @group.size
+        @satisfied_at = @group.size - num # trailing children that can be zero length affect when we are satisfied.        
+        @status = (@satisfied_at==@index) ? :D : :O
+        self
+      end
+      def inspect; _inspect '[',']'; end
+      def finalize; _advance if @child; self; end
+      def prune!; super %w(@group @index @satisfied_at @status @stop_here @kleene) end      
+      def expecting # cleanup @fixme @todo.  this was some genetic programming
+        return [] if self.status == :>
+        child = @child || self.last # might be nil if nothing was grabbed
+        expecting = []
+        if (child) # if we were able to parse at least one token
+          if ((index=@index-1) >= 0) # go backwards, reporting expected from any kleene closures
+            (index..0).each do
+              if (self[index].kleene) # kleeneup to use find_index
+                expecting |= self[index].expecting
+              end
+            end
+          end          
+          expecting += child.expecting unless child.status == :> # report the expecting tokens from the current symbol ()
+        end
+        index = size
+        # index = child ? ( @index + 1 ) : @index # whether or not we got to a token,
+        #index = @index
+        if ((!child or child.kleene or child.status.satisfied?) and @group and index < @group.size)
+          begin # go forward reporting the expecting from any kleene closures
+            expecting |= @group[index].expecting
+            break unless @group[index].kleene
+          end while((index+=1)<@group.size)
+        end  
+        expecting << "an end to the phrase" if (index == @group.size) 
+        expecting
+      end
+      def _advance
+        # @child.prune! unless @child.kleene #@todo
+        self << remove_instance_variable('@child') # child must be :> (if the child was :D we should keep it)
+        case (@index += 1) 
+          when @stop_here     then (:>) # iff we just finished the last child (there is no lookahead note 5)
+          when @satisfied_at  then (:D)
+          else                     (:O)
+        end
+      end      
+      def match token
+        while true
+          @child ||= @group[@index].dereference; # @group[@index] = nil; save it for prune
+          child_prev_status  = @child.status
+          child_status       = @child.match token
+          self._status = case child_status
+            when :> then _advance 
+            when :O then :O              
+            when :D then ((@index+1)>=@satisfied_at) ? :D : :O
+            when :C 
+              if (child_prev_status == :D)
+                self._status = _advance
+                next if @status.accepting?
+                :C
+              else 
+                :C
+              end
+            else; raise GorillaException.new('symbol returned bad status')
+          end # case
+          break; # break out of infinite loop
+        end # infinite loop
+        @status
+      end # def match
+    end # Sequence
+    class MoreOneOff
+      Grammar.register_shorthand :more, self
+      def self.construct_from_shorthand(a,*b); Infinity; end
+    end
+    class RangeOf < Array
+      alias_method :builtin_equality, :==
+      include CanParse, NonTerminalSymbol, PipeHack
+      [:zero_or_more,:one_or_more,:zero_or_one,:one,:range_of].each do |name|
+        Grammar.register_shorthand name, self
+      end
+      def self.construct_from_shorthand name, *args; self.new name, args; end
+      def initialize name, args
+        unless args.size > 0
+          raise GrammarGrammarException.new "Arguments must be non-zero length" 
+        end
+        @range = name.instance_of?(Range) ? name : case name
+          when :zero_or_more  then (0..Infinity)
+          when :one_or_more   then (1..Infinity)
+          when :zero_or_one   then (0..1)
+          when :one           then (1..1)
+          when :range_of         then args.shift
+          else raise UsageFailure.new(%{invalid name string "#{name}"})
+        end
+        raise UsageFailure.new("must be range") unless @range.instance_of? Range
+        @group = args.map{|x| GorillaSymbol.factory x }
+      end
+      attr_reader :range
+      attr_accessor :is_pipe_hack   
+      def reinit_for_parse
+        super
+        @group.each{ |x| x.reinit_for_parse; @unkleene = true unless x.kleene }
+        @kleene = @range.begin == 0 || ! @unkleene
+        @status = @kleene ? :D : :O         
+        @frame_prototype = Marshal.dump @group
+        _reframe
+        self
+      end
+      def prune!; super %w(@frame @frame_prototype @range @group @kleene @unkleene) end           
+      def << jobber # for PipeHack. code smell (:note 1)
+        if kind_of?(ParseTree) then super jobber
+        else; @group << GorillaSymbol.factory(jobber); end
+      end
+      def expecting; (@frame || @group || []).map{ |x| x.expecting }.flatten end
+      def inspect; 
+        _inspect '(',')',[@name ? %{:#{@name}} : nil , '(', @range.to_s.gsub('..Infinity',' or more'),'):'].compact.join
+      end
+      def match token
+        @status = nil
+        statii = Hash.new(){ |h,k| h[k] = [] }
+        @frame.each { |symbol| status = symbol.match(token); statii[status] << symbol }
+        if statii[:C].size == @frame.size then @status = :C
+        else case statii[:>].size
+          when 2..Infinity then raise AmbiguousGrammar.new(:parse => self, :children => statii[:>] )
+          when 1 then @status = _advance(statii[:>][0]) 
+          when 0 #fallthru
+        end end
+        # past this point we know that zero are :> and not all are :C, so some must be :O or :D
+        if @status.nil?
+          @frame.delete_if{ |x| ! x.status.accepting? }
+          @status = @frame.select{|x| x.status == :D }.count == @frame.size ? :D : :O
+        end
+        @status
+      end
+      def _advance object
+        self << object # (object.kleene ? object : object.prune!)
+        case size
+          when @range.end then        :>
+          when @range then  _reframe; :D
+          else;             _reframe; :O
+        end
+      end
+      def _reframe; @frame = (Marshal.load @frame_prototype).map{|x| x.kind_of?(SymbolReference) ? x.dereference : x}; end
+      ## @fixme this is waiting for unparse()
+      def self.join list, conj1, conj2, &block
+        list.map!(&block) if block
+        case list.size
+        when 0 then ''
+        when 1 then list[0]
+        else
+          joiners = ['',conj2]
+          joiners += Array.new(list.size-2,conj1) if list.size >= 3
+          list.zip(joiners.reverse).flatten.join 
+        end
+      end
+    end # RangeOf 
+  end
+end
+# note 1 having grammar nodes as parse tree nodes.  is it code smell?
+# note 3 (resolved - we use them now) consider getting rid of unused base classes
+# note 5 peeking isn't even used at this point
+# note 6 you might use to_s for unparse
+# note 7 todo: descention from regexp to string or vice versa,
+# note 8 one day we might have set-like RangeOfs that .., note 9 rangeof forks, sequence just inits group