genmachine 0.0.0 → 0.0.1

data/VERSION

@@ -1 +1 @@
-0.0.0
+0.0.1

data/bin/genmachine

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'genmachine'
+
+BANNER = "Usage: genmachine [options] STATE_TABLE_FILES"
+
+generators = GenMachine.generators
+languages = generators.keys
+
+options = {}
+opts = OptionParser.new do |opts|
+  opts.version = '0.0.1'
+  opts.banner = BANNER
+  opts.on('-c', '--classname NAME',
+          "Class/module/function name for generated library code "+
+          "(default STATE_TABLE_FILE)") do |v|
+    options[:classname] = v
+  end
+  opts.on('-l', '--language LANGUAGE',
+          "Language to generate code for- currently one of [#{languages.join(',')}] " +
+          "(default #{languages.first})") do |v|
+    options[:language] = v.to_underscored.to_sym
+  end
+  opts.on('-t', '--test-with FILE',
+          "Try parsing the specified file after generating the parser "+
+          "(default STATE_TABLE_FILE.gmtest if it exists)") do |v|
+    options[:test_file] = v
+  end
+  opts.on('-e', '--[no-]executable',
+          "Generate an executable parser (default true)") do |v|
+    options[:executable] = v
+  end
+  opts.on('-o', '--output-dir DIR',
+          "Output directory for generated file(s) (default ./)") do |v|
+    options[:output_dir] = v
+  end
+end
+
+files = opts.parse(ARGV)
+if files.size < 1
+  $stderr.puts BANNER
+  exit 1
+end
+
+file_base = files[0].chomp(File.extname(files[0]))
+name_base = File.basename(file_base)
+options[:executable] ||= true
+options[:language]   ||= languages.first
+options[:classname]  ||= name_base.capitalize + 'Parser'
+options[:test_file]  ||= files[0] + '.gmtest'
+options[:output_dif] ||= './'
+options[:class_fname]  = options[:classname].to_underscored + '.rb'
+options[:exe_fname]    = name_base.to_underscored
+
+unless languages.include? options[:language]
+  $stderr.puts "I don't know how to generate a parser in '#{options[:language]}' - try one of [#{languages.join(',')}]"
+  exit 2
+end
+
+#spec_parser = GenMachine::SpecParser.new(files)
+#options[:spec_ast] = spec_parser.build
+options[:spec_ast] = []
+gen = generators[options[:language]].new(options)
+gen.generate_class
+gen.generate_executable
+gen.run_test

data/genmachine.gemspec

@@ -0,0 +1,68 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{genmachine}
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Joseph Wecker"]
+  s.date = %q{2011-08-11}
+  s.default_executable = %q{genmachine}
+  s.description = %q{Takes a state table where the following are defined: state, input+conditions, accumulate-action, pre-transition-actions, and transition-to. It takes that state table and generates very fast parsers. Similar to Ragel. Currently only outputs pure Ruby.}
+  s.email = %q{joseph.wecker@gmail.com}
+  s.executables = ["genmachine"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/genmachine",
+    "genmachine.gemspec",
+    "lib/genmachine.rb",
+    "lib/genmachine/char_set.rb",
+    "lib/genmachine/generator.rb",
+    "lib/genmachine/generators/helpers/general.rb",
+    "lib/genmachine/generators/helpers/ruby.rb",
+    "lib/genmachine/generators/ruby.rb",
+    "lib/genmachine/generators/templates/ruby/lib.erb.rb",
+    "lib/genmachine/spec_parser.rb",
+    "test/helper.rb",
+    "test/test_genmachine.rb"
+  ]
+  s.homepage = %q{http://github.com/josephwecker/genmachine}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.6.2}
+  s.summary = %q{Generates parsers based on a fancy state-table}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/lib/genmachine/char_set.rb

@@ -0,0 +1,59 @@
+module GenMachine
+  class CharSet
+    attr_accessor :kind
+    def initialize(kind = :include)
+      @kind = kind
+      @include_intervals = []
+      @include_any = false
+    end
+
+    def inspect() for_conditional.inspect end
+    def [](k) for_conditional[k] end
+
+    def for_conditional
+      if @include_any
+        return {:kind=>@kind, :ranges=>[:any]}
+      else
+        ivals = @include_intervals.map do |a,b|
+          a == b ? a : [a,b]
+        end
+        return {:kind=>@kind, :ranges=>ivals}
+      end
+    end
+
+    def +(val) self.send(:<<,val) end
+    def <<(val)
+      if val.is_a?(String) && val =~ /([^-])-([^-])/
+          include_range($1,$2)
+      elsif val.is_a?(Range)
+        include_range(val.first, val.last)
+      elsif val == :any
+        @include_any = true
+      else include_char(val) end
+    end
+
+    def include_char(char) include_range(char,char) end
+
+    def include_range(from, to)
+      from = from.utf8_chars[0] if from.is_a?(String)
+      to = to.utf8_chars[0] if to.is_a?(String)
+      @include_intervals << [from,to].sort
+      if @include_intervals.length > 1
+        @include_intervals.sort!
+        merged = []
+        curr_a, curr_b = @include_intervals.shift
+        @include_intervals.each_with_index do |ab,i|
+          a,b = ab
+          if a <= (curr_b+1)
+            curr_b = [curr_b,b].max
+          else
+            merged << [curr_a, curr_b]
+            curr_a,curr_b = ab
+          end
+        end
+        merged << [curr_a, curr_b]
+        @include_intervals = merged
+      end
+    end
+  end
+end

data/lib/genmachine/generator.rb

@@ -0,0 +1,38 @@
+module GenMachine
+  module Generators
+    module Generator
+      def initialize(opts={})
+        opts ||= {}
+        @spec_ast       = opts.delete(:spec_ast)
+        @gen_executable = opts.delete(:executable) || false
+        @classname      = opts.delete(:classname)  || 'MiscParser'
+        @test_file      = opts.delete(:test_file)
+        @output_dir     = opts.delete(:output_dir) || './'
+        @class_fname    = opts.delete(:class_fname)|| @classname.to_underscored
+        @exe_fname      = opts.delete(:exe_fname)  || @classname.sub(/parser$/i,'').to_underscored
+        raise ArgumentError, "Must include the table specification data (:spec_ast)" if @spec_ast.nil?
+      end
+
+      def generate_class
+        puts "Generate class (#{@classname}): #{@class_fname}"
+        #f = File.new(@class_fname, 'w+')
+        #f.write(@libraries[language].result(binding))
+        #f.close
+      end
+
+      def generate_executable
+        return unless @gen_executable
+        puts "Generate executable: #{@exe_fname}"
+        #f = File.new(@exe_fname, 'w+')
+        #f.write(@executables[language].result(binding))
+        #f.chmod(0755)
+        #f.close
+      end
+
+      def run_test
+        return if @test_file.nil?
+        puts "Run test"
+      end
+    end
+  end
+end

data/lib/genmachine/generators/helpers/general.rb

@@ -0,0 +1,62 @@
+class String
+  def to_underscored
+    self.gsub(/::/, '/').
+    gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
+    gsub(/([a-z\d])([A-Z])/,'\1_\2').
+    gsub(/[^a-zA-Z\d_]/,'_').
+    downcase
+  end
+
+  def to_utf8_char_array
+    self.unpack("U*")
+  end
+end
+
+module GenMachine
+  module Helpers
+    module General
+      def accumulates?(states)
+        states.each do |name,clauses|
+          clauses.each do |c|
+            c[:exprs].each do |e|
+              return true if e =~ /<<s($|[^a-zA-Z0-9_])/
+            end
+          end
+        end
+        return false
+      end
+
+      def accumulators(states)
+        accs = {}
+        states.each do |name,clauses|
+          clauses.each do |c|
+            exprs = c[:exprs].dup
+            exprs << c[:acc].dup
+            exprs.each do |e|
+              if e =~ /^([a-zA-Z_][a-zA-Z0-9_]*)?\s*<<\s*(<?)([a-zA-Z_][a-zA-Z0-9_]*)?$/
+                accs[$1]=true unless ($1.nil? or $1 == '' or $1 == 'p' or $1 == 's')
+                accs[$3]=true unless ($3.nil? or $3 == '' or $3 == 'p' or $3 == 's')
+              end
+            end
+          end
+        end
+        return accs.keys
+      end
+
+      def makes_calls?(states)
+        # TODO: implement
+        false
+      end
+
+      def eof_state?(states)
+        states.each{|name,clauses| return true if name=='{eof}'}
+        return false
+      end
+
+      def eof_clause?(clauses)
+        clauses.each{|c| return true if c[:cond].include?('eof')}
+        return false
+      end
+    end
+  end
+end

data/lib/genmachine/generators/helpers/ruby.rb

@@ -0,0 +1,121 @@
+module GenMachine
+  module Helpers
+    module Ruby
+      def rb_conditional(clause,states,clauses)
+        has_eof_state = eof_state?(states) || eof_clause?(clauses)
+        out = ''
+        if clause[:cond].size > 0
+          out += '('+clause[:cond].join(' || ')+')'
+          out += "&& (#{rb_charset_cond(clause[:input],has_eof_state)})" unless clause[:input].nil?
+        else
+          out += rb_charset_cond(clause[:input],has_eof_state,',')
+        end
+        return rb_vars(out)
+      end
+
+      def rb_charset_cond(input,has_eof_state,sep='||')
+        return 'true' if input.nil?
+        outs = []
+        sep = '||' if input[:kind] == :exclude
+        input[:ranges].each do |range|
+          if input[:kind] == :include
+            if range.is_a? Array
+              outs << "nl?" if (range[0] <= 0x0a) && (range[1] >= 0x0a)
+              outs << "space?" if (range[0] <= 0x20) && (range[1] >= 0x20)
+              outs << "(c>#{range[0]-1}&&c<#{range[1]+1})"
+            else
+              outs << case range
+                      when 0x0a; 'nl?'
+                      when 0x20; 'space?'
+                      when :any; has_eof_state ? 'true' : 'c!=:eof'
+                      else "c==#{range}" end
+            end
+          end
+        end
+        out = outs.join(sep)
+        out = '!('+out+')' if input[:kind] == :exclude
+        return out
+      end
+
+      def rb_commands(clause,currstate)
+        cmds = []
+        cmds += rb_simple_acc_commands(clause[:acc])
+        clause[:exprs].each do |expr|
+          if expr.include? '<<'
+            cmds += rb_acc_commands(expr)
+          else
+            cmds << rb_vars(expr.strip)
+          end
+        end
+        cmds += rb_transition_commands(clause[:next],currstate)
+        return cmds.join('; ')
+      end
+
+      def rb_transition_commands(st,currstate)
+        st = st.strip.split(';').map{|s|s.strip}
+        out = []
+        add_next = false
+        st.each do |s|
+          case
+          when s =~ /^([^\(\[]+)(?:\[([^\]]*)\])?\(([^\)]*)\)$/   # Call another group
+            funname = $1
+            rename = $2
+            params = $3.split(',').map{|p|rb_vars(p)}
+            params << 's'
+            params << "'#{rename}'" unless (rename.nil? or rename.strip=='')
+            out << "state=#{funname}(#{params.join(',')})"
+            add_next = true
+          when s =~ /^<done>$/
+            out << "return(s)"
+            add_next = false
+          when s =~ /^<([^>]+)>$/
+            out << "return(#{rb_vars($1)})"
+            add_next = false
+          when s =~ /^(:[a-zA-Z0-9_:-]+)$/
+            out << "state='#{$1}'" unless currstate == $1
+            add_next = true
+          else
+            out << s
+          end
+        end
+        out << 'next' if add_next
+        return out
+      end
+
+      def rb_vars(str)
+        str.tr('$','@').gsub /(:[a-zA-Z0-9_:-]+)/, '\'\1\''
+      end
+
+      def rb_simple_acc_commands(acc_phrase)
+        case
+        when (acc_phrase.nil? or acc_phrase == ''); return ['@fwd=true']
+        when acc_phrase.strip == '<<'; return []
+        when acc_phrase.strip =~ /^([a-zA-Z_][a-zA-Z0-9_]*)\s*<<\s*$/
+          return ["#{rb_vars(acc_phrase.strip)}c"]
+        else raise("Can't figure out your accumulator statement: #{acc_phrase}")
+        end
+      end
+
+      def rb_acc_commands(acc_phrase)
+        case
+        when (acc_phrase.nil? or acc_phrase == ''); return ['@fwd=true']
+        when acc_phrase.strip == '<<'; return []
+        when acc_phrase.strip =~ /^([a-zA-Z_][a-zA-Z0-9_]*)\s*<<\s*(<?)([a-zA-Z_][a-zA-Z0-9_]*)$/
+          into = $1
+          value = $3
+          clear_it = $2 == '<'
+          into = rb_vars(into)
+          value = rb_vars(value)
+          if clear_it
+            out = ["(#{into}<<#{value} if #{value}.size>0)"]
+            out << "#{value}=''"
+          else
+            out = ["#{into}<<#{value}"]
+          end
+          return out
+        else raise("Can't figure out your accumulator statement: #{acc_phrase}")
+        end
+      end
+    end
+  end
+end

data/lib/genmachine/generators/ruby.rb

@@ -0,0 +1,8 @@
+module GenMachine
+  module Generators
+    class RubyGenerator
+      include Generator
+      GENMACHINE_TARGET = 'ruby'
+    end
+  end
+end

data/lib/genmachine/generators/templates/ruby/lib.erb.rb

@@ -0,0 +1,175 @@
+require 'strscan'
+$KCODE="U"
+
+module <%= classname %>
+  def self.parse(str) Parser.new(str).parse end
+  def self.parse_file(fname) Parser.new(IO.read(fname)).parse end
+
+  class Node
+    attr_accessor :name, :children, :start_line, :start_pos, :end_line, :end_pos
+    def initialize(name='node',line=:unknown,pos=:unknown)
+      @name = name
+      @children = []
+      @start_line = line
+      @start_pos = pos
+      @end_line = :unknown
+      @end_pos = :unknown
+    end
+    def <<(val) @children<<val end
+  end
+
+  class Parser < StringScanner
+    def init(source, opts={})
+      opts ||= {}
+      super ensure_encoding(source)
+      @global = {}
+    end
+
+    def ensure_encoding(source)
+      if defined?(::Encoding)
+        if source.encoding == ::Encoding::ASCII_8BIT
+          b = source[0, 4].bytes.to_a
+          source =
+            case
+            when b.size>=4 && b[0]==0 && b[1]==0 && b[2]==0
+              source.dup.force_encoding(::Encoding::UTF_32BE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[0]==0 && b[2]==0
+              source.dup.force_encoding(::Encoding::UTF_16BE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[1]==0 && b[2]==0 && b[3]==0
+              source.dup.force_encoding(::Encoding::UTF_32LE).encode!(::Encoding::UTF_8)
+            when b.size>=4 && b[1]==0 && b[3]==0
+              source.dup.force_encoding(::Encoding::UTF_16LE).encode!(::Encoding::UTF_8)
+            else source.dup end
+        else source = source.encode(::Encoding::UTF_8) end
+        source.force_encoding(::Encoding::ASCII_8BIT)
+      else
+        b = source
+        source =
+          case
+          when b.size >= 4 && b[0] == 0 && b[1] == 0 && b[2] == 0; JSON.iconv('utf-8', 'utf-32be', b)
+          when b.size >= 4 && b[0] == 0 && b[2] == 0; JSON.iconv('utf-8', 'utf-16be', b)
+          when b.size >= 4 && b[1] == 0 && b[2] == 0 && b[3] == 0; JSON.iconv('utf-8', 'utf-32le', b)
+          when b.size >= 4 && b[1] == 0 && b[3] == 0; JSON.iconv('utf-8', 'utf-16le', b)
+          else b end
+      end
+      return source
+    end
+
+    def parse
+      reset
+      @line = 1
+      @pos = 1
+      @leading = true
+      @indent = 0
+      @ast = <%= @table[0][0] %>
+      return @ast
+    end
+
+    private
+
+    def error(msg)
+      $stderr.puts "#{msg} | line: #{@line} | char: #{@pos}"
+    end
+
+    def global_state(c)
+      # Unicode newline characters & combinations
+      # Plus leading space for indents.
+      # Also tracks line and position for the AST
+      @last_is_newline = @last_is_space = false
+      case c
+      when 0x0b, 0x0c, 0x85, 0x2028, 0x2029
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x0a
+        nc = peek(1).unpack('U')[0]
+        if nc == 0x0d then getch; c = 0x0a0d end
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x0d
+        nc = peek(1).unpack('U')[0]
+        if nc == 0x0a then getch; c = 0x0d0a end
+        @last_is_newline = true; @line += 1; @pos = 1
+        @leading = true; @indent = 0
+      when 0x20
+        @indent += 1 if @leading
+        @last_is_space = true; @pos += 1
+      else @leading = false; @pos += 1 end
+      return @last_c = c
+    end
+
+    def nl?() return @last_is_newline end
+    def space?() return @last_is_space end
+
+    def nextchar
+      if @fwd then @fwd = false; return @last_c
+      else
+        c = getch
+        if c.nil?
+          c = :eof
+          @last_is_space = @last_is_newline = false
+          return @last_c = c
+        end
+        return global_state(c.unpack('U')[0])
+      end
+    end
+
+    def eof?() return @last_c == :eof end
+
+<%- @table.each do |name, args, cmds, first_state, states| -%>
+  <%- args << "p=nil" -%>
+  <%- args << "name='#{name}'" -%>
+    def <%= name %>(<%= args.join(',') %>)
+      <%- cmds.each do |c| -%>
+      <%= rb_vars(c) %>
+      <%- end -%>
+      state='<%= first_state %>'
+      <%- if states.size > 1 or accumulates?(states) or makes_calls?(states) -%>
+      s = Node.new(name,@line,@pos)
+      <%- end -%>
+      <%- accumulators(states).each do |_acc| -%>
+      <%= _acc %> ||= ''
+      <%- end -%>
+      loop do
+        c = nextchar
+        <%- has_fallthru = false -%>
+        <%- if eof_state?(states) -%>
+        state = '{eof}' if c==:eof
+        <%- end -%>
+        case state
+        <%- states.each do |st_name, clauses| -%>
+        when '<%= st_name %>'
+          <%- if clauses.size > 1 -%>
+            case
+            <%- clauses.each_with_index do |clause,i| -%>
+              <%- cond = rb_conditional(clause,states,clauses) -%>
+                <%- if cond == 'true' -%>
+            else <%= rb_commands(clause,st_name) %>
+                  <%- break -%>
+                <%- else -%>
+            when <%= cond %>; <%= rb_commands(clause,st_name) %>
+                <%- has_fallthru = true if i == clauses.size-1 -%>
+                <%- end -%>
+            <%- end -%>
+            end
+          <%- else -%>
+            <%- cond = rb_conditional(clauses[0],states,clauses) -%>
+            <%- if cond == 'true' -%>
+            <%= rb_commands(clauses[0],st_name) %>
+            <%- else -%>
+            if <%= cond %>
+              <%= rb_commands(clauses[0],st_name) %>
+            end
+            <%- end -%>
+          <%- end -%>
+        <%- end -%>
+        end
+        <%- if has_fallthru -%>
+        error("Unexpected #{c}")
+        @fwd = true
+        return
+        <%- end -%>
+      end
+    end
+<%- end -%>
+  end
+end

data/lib/genmachine/spec_parser.rb

@@ -0,0 +1,130 @@
+module GenMachine
+  # This is a quick and dirty parser used for bootstrapping. Which means
+  # it'll eventually be replaced when the real parser is written as a
+  # genmachine table.
+  class SpecParser
+    def initialize(files)
+      @table = []
+      @files = files
+      @libraries = {}
+      @executables = {}
+      template_base = File.expand_path(File.dirname(__FILE__))+'/templates/'
+      LANGUAGES.each do |lang|
+        tbase = template_base + lang.to_s + '/'
+        @libraries[lang] = ERB.new(IO.read(tbase+'library.erb.rb'),nil,'-')
+        @executables[lang] = ERB.new(IO.read(tbase+'executable.erb'),nil,'-')
+      end
+    end
+
+    def build
+      c_name = c_args = c_cmds = c_first_state = c_states = nil
+      new_fun = false
+      @files.each do |fname|
+        File.new(fname,'r').each_with_index do |line, line_no|
+          line = line.strip
+          det = line[0..0]
+          if det == '|' or det == ':'
+            re = (det=='|' ? '\|' : det) + '(?: |$)'
+            cols = line.split(/#{re}/,-1)[1..-1].map{|c| c.strip}
+            if det=='|' && cols[0].include?('(')
+              new_fun = true
+              unless c_name.nil?
+                @table << [c_name, c_args, c_cmds, c_first_state, process_states(c_states)]
+              end
+              parts = cols[0].split('(')
+              c_name = parts.shift.underscore
+              c_args = parts.join('(').sub(/\)$/,'').split(',')
+              c_states = []
+              c_cmds = (cols[3]||'').split(';')
+              c_first_state = cols[4]
+              # TODO: error if cols[1] or cols[2] have anything
+            elsif det == ':' && new_fun
+              c_args += cols[0].sub(/\)$/,'').split(',')
+              c_cmds += (cols[3]||'').split(';')
+              c_first_state += (cols[4]||'')
+            elsif det == '|'
+              new_fun = false
+              conditionals, inputs = parse_input(cols[1])
+              c_states << {:name  => cols[0],
+                           :input => inputs,
+                           :cond  => conditionals,
+                           :acc   => cols[2],
+                           :exprs => (cols[3]||'').split(';'),
+                           :next  => cols[4]}
+            elsif det == ':' && (c_states.size > 0)
+              conditionals, inputs = parse_input(cols[1],c_states[-1][:input])
+              c_states[-1][:name] += (cols[0]||'')
+              c_states[-1][:input] = inputs
+              c_states[-1][:cond] += conditionals
+              c_states[-1][:acc]  += cols[2]
+              c_states[-1][:exprs]+= (cols[3]||'').split(';')
+              c_states[-1][:next] += cols[4]
+            end
+          end
+        end
+        unless c_name.nil?
+          @table << [c_name, c_args, c_cmds, c_first_state, process_states(c_states)]
+        end
+      end
+      pp @table
+    end
+
+    # consolidate same-name states and (eventually) combine / optimize where
+    # appropriate.
+    def process_states(instates)
+      outstates = {}
+      instates.each do |inst|
+        name = inst.delete(:name)
+        outstates[name] ||= []
+        outstates[name] << inst
+      end
+      return outstates
+    end
+
+    def parse_input(val,inputs=nil)
+      iters = 0
+      conds = []
+      while val.strip.length > 0 && iters < 100
+        case
+        when val =~ /--+/um
+          val.sub!($&,'')
+        when val =~ /\s*\{([^\}]+)\}\s*/um
+          conds << $1
+          val.sub!($&, '')
+        when val =~ /\s*\[\^([^\]]+)\]\s*/um
+          inputs ||= CharSet.new(:exclude)
+          parse_combine_ranges($1, inputs)
+          val.sub!($&, '')
+        when val =~ /\s*\[([^\]]+)\]\s*/um
+          inputs ||= CharSet.new(:include)
+          parse_combine_ranges($1, inputs)
+          val.sub!($&, '')
+        when val =~ /^\s*\./um
+          inputs ||= CharSet.new(:include)
+          inputs << :any
+          val.sub!($&, '')
+        end
+        iters += 1
+      end
+      return conds, inputs
+    end
+
+    ESCAPES = {'\t' => "\t", '\n' => "\n",
+               '\r' => "\r", '\f' => "\f",
+               '\b' => "\b", '\a' => "\a",
+               '\e' => "\e", '\s' => " ",
+               '\[' => '[',  '\]' => ']'}
+    def parse_combine_ranges(raw, input)
+      raw.gsub!(/\\[tnrfbaes\[\]]/){|m| ESCAPES[m]}
+      if raw =~ /((?:.-.)*)((?:.)*)/um
+        ranges = $1
+        singles = $2
+        if ranges.length > 0
+          _, range, ranges = ranges.partition /.-./um
+          input << range
+        end while ranges.length > 0
+        singles.scan(/./um).each{|s| input << s}
+      end
+    end
+  end
+end

data/lib/genmachine.rb

@@ -1,8 +1,22 @@
+require 'genmachine/spec_parser'
+require 'genmachine/generator'
+require 'genmachine/generators/helpers/general'
+
+Dir[File.join(File.dirname(__FILE__),'genmachine','generators','*.rb')].each do |fname|
+  name = File.basename(fname)
+  require "genmachine/generators/#{name}"
+end
+
 module GenMachine
-  class Builder
-    def initialize(files)
-      @files = files
-      @table = []
+  class << self
+    def generators
+      Generators.constants.reduce({}) do |langs,const|
+        klass = Generators.const_get(const)
+        if klass.const_defined?('GENMACHINE_TARGET')
+          langs[klass::GENMACHINE_TARGET] = klass
+        end
+        langs
+      end
     end
   end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: genmachine
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: 
   segments: 
   - 0
   - 0
-  - 0
-  version: 0.0.0
+  - 1
+  version: 0.0.1
 platform: ruby
 authors: 
 - Joseph Wecker
@@ -15,8 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-10 00:00:00 -07:00
-default_executable: 
+date: 2011-08-11 00:00:00 -07:00
+default_executable: genmachine
 dependencies: 
 - !ruby/object:Gem::Dependency 
   requirement: &id001 !ruby/object:Gem::Requirement 
@@ -80,8 +80,8 @@ dependencies:
   type: :development
 description: "Takes a state table where the following are defined: state, input+conditions, accumulate-action, pre-transition-actions, and transition-to. It takes that state table and generates very fast parsers. Similar to Ragel. Currently only outputs pure Ruby."
 email: joseph.wecker@gmail.com
-executables: []
-
+executables: 
+- genmachine
 extensions: []
 
 extra_rdoc_files: 
@@ -94,7 +94,16 @@ files:
 - README.rdoc
 - Rakefile
 - VERSION
+- bin/genmachine
+- genmachine.gemspec
 - lib/genmachine.rb
+- lib/genmachine/char_set.rb
+- lib/genmachine/generator.rb
+- lib/genmachine/generators/helpers/general.rb
+- lib/genmachine/generators/helpers/ruby.rb
+- lib/genmachine/generators/ruby.rb
+- lib/genmachine/generators/templates/ruby/lib.erb.rb
+- lib/genmachine/spec_parser.rb
 - test/helper.rb
 - test/test_genmachine.rb
 has_rdoc: true