import_everything 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,5 @@
+*.sw?
+.DS_Store
+coverage
+rdoc
+pkg

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Mike Harris
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,18 @@
+= import_everything
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+   bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Mike Harris. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,49 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "import_everything"
+    gem.summary = %Q{import everything}
+    gem.description = %Q{import everything}
+    gem.email = "mharris717@gmail.com"
+    gem.homepage = "http://github.com/mharris717/import_everything"
+    gem.authors = ["Mike Harris"]
+    gem.add_development_dependency "rspec"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "import_everything #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/import_everything/ext.rb

@@ -0,0 +1,92 @@
+class Object
+  def local_methods
+    res = methods - 7.methods - ''.methods - Object.new.methods
+    res.sort
+  end
+end
+
+class String
+  def blank?
+    strip == ''
+  end
+  def present?
+    !blank?
+  end
+end
+
+module MethodLogging
+  def log_method(name)
+    alias_method "unlogged_#{name}", name
+    define_method(name) do |*args|
+      res = send("unlogged_#{name}",*args)
+      puts "#{name} returned #{res.inspect} from #{args.inspect}"
+      res
+    end
+  end
+  def def_logging_method(name,&b)
+    define_method(name) do |*args|
+      res = instance_eval()
+    end
+  end
+end
+
+class Hash
+  def self.from_keys_and_values(ks,vs)
+    raise "size not equal #{ks.size} #{vs.size}" unless ks.size == vs.size
+    ks.zip(vs).inject({}) { |h,a| h.merge(a[0] => a[1]) }
+  end
+end
+
+class String
+  def without_quotes
+    if self =~ /^['"].*['"]$/
+      self[1..-2]
+    else
+      self
+    end
+  end
+  def to_num_if
+    if self =~ /^\d+$/
+      self.to_i
+    elsif self =~ /^[\d\.]+$/
+      self.to_f
+    else
+      self
+    end
+  end
+  def fixed_obj
+    strip.without_quotes.to_num_if
+  end
+end
+
+class Object
+  def fixed_obj
+    self
+  end
+end
+
+module Enumerable
+  def make_last
+    h = group_by { |x| !!yield(x) }
+    (h[false]||[]) + (h[true]||[])
+  end
+  def make_first
+    h = group_by { |x| !!yield(x) }
+    (h[true]||[]) + (h[false]||[])
+  end
+end
+
+class Hash
+  def cleaned_hash_values
+    map_value { |v| v.fixed_obj }
+  end
+end
+
+class Object
+  def first_responding(*methods)
+    methods.flatten.each { |x| return send(x) if respond_to?(x) }
+    #raise "none respond"
+    #raise local_methods.inspect
+    nil
+  end
+end

data/lib/import_everything/parsers/csv_parser.rb

@@ -0,0 +1,16 @@
+module ImportEverything
+  class CsvParser < Parser
+    fattr(:delimiter) { "," }
+    def value_hashes
+      require 'csv'
+      res = []
+      CSV.parse(str, :headers => true, :col_sep => delimiter) do |row|
+        res << row.to_hash
+      end
+      res
+    end
+    def required_fields
+      []
+    end
+  end
+end

data/lib/import_everything/parsers/determine_type.rb

@@ -0,0 +1,36 @@
+module ImportEverything
+  class DetermineType
+    include FromHash
+    def self.get_filename(file)
+      if file.kind_of?(String)
+        ''
+      else
+        file.first_responding(:original_filename, :filename, :path, :name)
+      end
+    end
+    fattr(:filename) { self.class.get_filename(file) }
+    fattr(:file) { open(filename) }
+    fattr(:ext) { filename.split(".").last.downcase }
+    def method_missing(sym,*args,&b)
+      if sym.to_s[-1..-1] == '='
+        self.addl_ops[sym.to_s[0..-2]] = args.first
+      else
+        super
+      end
+    end
+    fattr(:addl_ops) { {} }
+    fattr(:parser_ops) { {:file => file}.merge(addl_ops) }
+    def parser_class
+      h = {'sqlite' => SqliteParser, 'sqlite3' => SqliteParser, 'csv' => CsvParser, 'xml' => XmlParser, 'sql' => SqlInsertParser, 'dmp' => SqlInsertParser, 'html' => TableParser}
+      h[ext].tap { |x| return x if x }
+      h.each do |k,klass|
+        return klass if ext =~ /^#{k}\d\d/
+      end
+      raise "no parser found for #{ext}"
+    end
+    def parser
+#      puts "parser ops is #{parser_ops.inspect}"
+      parser_class.new(parser_ops)
+    end
+  end
+end

data/lib/import_everything/parsers/line_parser.rb

@@ -0,0 +1,13 @@
+module ImportEverything
+  class LineParser
+    def value_hashes
+      [value_hash]
+    end
+    fattr(:cleaned_value_hashes) do
+      value_hashes.map { |row| row.cleaned_hash_values }
+    end
+    def row_hashes
+      value_hashes.map { |x| {:table => table, :values => x} }
+    end
+  end
+end

data/lib/import_everything/parsers/module_methods.rb

@@ -0,0 +1,18 @@
+module ImportEverything
+  def self.get_parser(ops)
+    dt = ImportEverything::DetermineType.new(ops)
+    dt.parser
+  end
+  def self.get_rows(ops)
+    get_parser(ops).cleaned_row_hashes
+  end
+  def self.each_row(ops,&b)
+    get_parser(ops).each_row(&b)
+  end
+  def self.each_table_and_rows(ops,&b)
+    get_parser(ops).each_table_and_rows(&b)
+  end
+  def self.preview(ops)
+    Preview.new(ops)
+  end
+end

data/lib/import_everything/parsers/parser.rb

@@ -0,0 +1,55 @@
+module ImportEverything
+  class Parser
+    fattr(:table) { 'some_table' }
+    include FromHash
+    fattr(:cleaned_value_hashes) do
+      value_hashes.map { |row| row.cleaned_hash_values }
+    end
+    fattr(:cleaned_row_hashes) do
+      row_hashes.map do |row_hash|
+        {:table => row_hash[:table], :values => row_hash[:values].cleaned_hash_values}
+      end
+    end
+  
+    fattr(:value_hashes) do
+      parsers.map { |x| x.value_hashes }.flatten
+    end
+    fattr(:row_hashes) do
+      if respond_to?(:parsers)
+        parsers.map { |x| x.row_hashes }.flatten
+      else
+        value_hashes.map { |x| {:table => table, :values => x} }
+      end
+    end
+    def line_parsers; parsers; end
+    fattr(:filename) { ImportEverything::DetermineType.get_filename(file) }
+    fattr(:file) { open(filename) }
+    fattr(:str) { file.read }
+    
+    def each_row
+      each_table_and_rows do |table,rows|
+        rows.each { |row| yield(table,row) }
+      end
+    end
+    def each_table_and_rows
+      cleaned_row_hashes.group_by { |x| x[:table] }.each do |table,rows|
+        values = rows.map { |x| x[:values] }
+        yield(table,values)
+      end
+    end
+    def table_rows_hash
+      res = {}
+      each_table_and_rows { |table,rows| res[table] = rows }
+      res
+    end
+  end
+  module ParserPreviewMod
+    fattr(:addl_required_fields) do
+      required_fields.select do |x| 
+        send(x).to_s.blank? 
+      end
+    end
+    fattr(:required_fields) { [] }
+  end
+  Parser.send(:include,ParserPreviewMod)
+end

data/lib/import_everything/parsers/preview.rb

@@ -0,0 +1,58 @@
+module ImportEverything
+  class DisplayTable
+    attr_accessor :rows, :table
+    include FromHash
+    fattr(:keys) do
+      rows.map { |x| x.keys }.flatten.uniq
+    end
+    def each_row_value_array
+      rows.each do |row|
+        vals = keys.map { |k| row[k] }
+        vals.each { |x| yield(x) }
+      end
+    end
+    def row_value_arrays
+      rows.map do |row|
+        keys.map { |k| row[k] }
+      end
+    end
+    def rows
+            #mylog "display_table", :rows => @rows
+            @rows
+    end
+    def to_hash
+      {:keys => keys, :row_arrays => row_value_arrays, :table => table}
+    end
+  end
+  class Preview
+    attr_accessor :ops
+    def initialize(ops)
+      self.ops = ops
+    end
+    fattr(:dt) { ImportEverything::DetermineType.new(ops) }
+    def parser
+      dt.parser
+    end
+    def addl_required_fields
+      parser.addl_required_fields
+    end
+    def ready?
+      addl_required_fields.empty?
+    end
+    def preview_table
+      DisplayTable.new(:rows => parser.cleaned_value_hashes)
+    end
+    def table; preview_table; end
+    def tables
+      parser.table_rows_hash.map { |table,rows| DisplayTable.new(:table => table, :rows => rows) }
+    end
+    def method_missing(sym,*args,&b)
+      if sym.to_s[-1..-1] == '='
+        self.ops[sym.to_s[0..-2]] = args.first
+        dt!
+      else
+        super
+      end
+    end
+  end
+end

data/lib/import_everything/parsers/sql_parser.rb

@@ -0,0 +1,79 @@
+module ImportEverything
+  class SqlInsertParser < Parser
+    fattr(:lines) do
+      str.split(/\n|;/).map { |x| x.strip }.select { |x| x.present? }
+    end
+    fattr(:probable_insert_lines) do
+      lines.select { |x| LineParser.probable_insert?(x) }
+    end
+    def get_insert_lines(lines)
+      if lines.empty?
+        []
+      elsif LineParser.comment?(lines.first)
+        get_insert_lines(lines[1..-1])
+      elsif LineParser.valid_insert?(lines.first)
+        [lines.first] + get_insert_lines(lines[1..-1])
+      elsif lines.size >= 2 && LineParser.valid_insert?(lines[0..1].join(" "))
+        [lines[0..1].join(" ")] + get_insert_lines(lines[2..-1])
+      elsif lines.size >= 3 && LineParser.valid_insert?(lines[0..2].join(" "))
+        [lines[0..2].join(" ")] + get_insert_lines(lines[2..-1])
+      else
+        get_insert_lines(lines[1..-1])
+      end
+    end
+    fattr(:insert_lines) do
+      get_insert_lines(lines)
+    end
+    fattr(:parsers) do
+      insert_lines.map { |ln| LineParser.new(:line => ln) }
+    end
+  
+    class LineParser < LineParser
+      attr_accessor :line
+      include FromHash
+      extend MethodLogging
+      class << self
+        fattr(:insert_into_regex) { /INSERT\s+INTO\s+(\S+)/i }
+        fattr(:columns_regex) { /\((.*)\)/i }
+        fattr(:values_regex) { /VALUES\s+\((.*)\)/i }
+        fattr(:full_regex) { /^\s*#{insert_into_regex}\s+#{columns_regex}\s+#{values_regex}/i }
+        fattr(:regex_hash) do
+          {:insert => insert_into_regex, :columns => columns_regex, :values => values_regex, :full => full_regex}
+        end
+        fattr(:comment_regex) do
+          /^(\/\*|#)/
+        end
+        def probable_insert?(str)
+          str = str.strip
+          res = (str =~ insert_into_regex) && !comment?(str)
+          !!res
+        end
+        def valid_insert?(str)
+          !!(str =~ full_regex)
+        end
+        def comment?(str)
+          !!(str =~ comment_regex)
+        end
+      end
+      fattr(:parsed_elements) do
+        raise "doesn't parse" unless line =~ self.class.full_regex
+        {:table => $1, :columns => $2, :values => $3}
+      end
+      fattr(:table) { parsed_elements[:table].without_quotes }
+      fattr(:columns) do
+        parsed_elements[:columns].split(",").map { |x| x.downcase.strip.without_quotes }
+      end
+      #log_method :fix_value
+      fattr(:values) do
+        require 'csv'
+        raw = parsed_elements[:values]
+        CSV.parse(raw).first.map { |x| x.fixed_obj }
+      end
+      fattr(:value_hash) do
+        Hash.from_keys_and_values(columns,values)
+      end
+    end
+  end
+end
+
+

data/lib/import_everything/parsers/sqlite_parser.rb

@@ -0,0 +1,46 @@
+module ImportEverything
+  class SqliteParser < Parser
+    fattr(:filfename) do
+      f = Tempfile.new('somedb.sqlite3')
+      f.binmode
+      f << file.read
+      f.path
+    end
+    fattr(:db) do
+      gem 'sqlite3-ruby'
+      require 'sqlite3'
+      #raise "#{filename} #{file.local_methods.inspect}"
+      res = SQLite3::Database.new(filename) 
+      #get_raw_tables(res)
+      #res
+    end
+    def get_raw_tables(db)
+      sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
+      res = db.execute(sql).flatten
+      mylog "sqlite", :res => res, :sql => sql
+      res
+    end
+    fattr(:raw_tables) { get_raw_tables(db) }
+    fattr(:tables) do
+      raw_tables.reject { |x| x =~ /^(schema|sqlite)/ }
+    end
+    fattr(:parsers) do
+      tables.map { |x| TableParser.new(:db => db, :table => x) }
+    end
+    def required_fields
+      []
+    end
+    class TableParser < LineParser
+      attr_accessor :db, :table
+      include FromHash
+      fattr(:execute_result) { db.execute2("SELECT * FROM #{table}") }
+      fattr(:columns) { execute_result.first }
+      fattr(:raw_rows) { execute_result[1..-1] }
+      fattr(:value_hashes) do
+        raw_rows.map do |row|
+          Hash.from_keys_and_values(columns,row)
+        end
+      end
+    end
+  end
+end

data/lib/import_everything/parsers/table_parser.rb

@@ -0,0 +1,5 @@
+module ImportEverything
+  class TableParser < CsvParser
+    fattr(:delimiter) { "\t" }
+  end
+end

data/lib/import_everything/parsers/xml_parser.rb

@@ -0,0 +1,38 @@
+module ImportEverything
+  class XmlParser < Parser
+    attr_accessor :root_path, :table_paths, :xyz
+    #fattr(:table_paths) { [['players','player'],['cities','city']] }
+    #fattr(:root_path) { 'top' }
+    fattr(:raw_doc) { require 'hpricot'; Hpricot(str) }
+    fattr(:doc) { raw_doc/root_path }
+    fattr(:parsers) do
+      table_paths.map do |table_desc|
+        docs = (doc/table_desc[0])
+        TableParser.new(:table => table_desc[0], :doc => docs, :row_path => table_desc[1])
+      end
+    end
+    def required_fields
+      [:root_path,:table_paths]
+    end
+  
+    class TableParser < Parser
+      attr_accessor :doc, :table, :row_path
+      include FromHash
+      def row_docs
+        doc/row_path
+      end
+      def parsers
+        row_docs.map { |doc| LineParser.new(:doc => doc, :table => table) }
+      end
+    end
+  
+    class LineParser < LineParser
+      attr_accessor :doc, :table
+      include FromHash
+      def value_hash
+        cs = doc.children.reject { |x| x.to_s.blank? }
+        cs.inject({}) { |h,el| h.merge(el.name => el.inner_text) }
+      end
+    end
+  end
+end

data/lib/import_everything.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'mharris_ext'
+require 'active_support'
+
+this_path = File.expand_path(File.dirname(__FILE__))
+require "#{this_path}/import_everything/ext"
+paths = Dir["#{this_path}/import_everything/**/*.rb"]
+paths = paths.make_first { |path| %w(parser.rb line_parser.rb).include?(File.basename(path)) }
+paths.each { |x| require x }

data/spec/bets.html

@@ -0,0 +1,3 @@
+Event Name	Market Type	Selection	Odds	Stake ($)	Market P&L ($)
+Eastern Conference	Winner	Orlando (to win)	+282	157.09	443.00
+Carib Stupidity	Carib Stupidity	Vegas Watch 107494 (to win)	+180	1.00	1.79

data/spec/csv_parser_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "ImportEverything" do
+  describe CsvParser do
+    describe 'players' do
+      before do
+        @parser = CsvParser.new(:filename => spec_file('players.csv'), :table => 'players')
+      end
+      it 'smoke' do
+        2.should == 2
+      end
+      it 'row count' do
+        @parser.value_hashes.should size(3)
+        @parser.cleaned_row_hashes.should size(3)
+      end
+      it 'cleaned hash' do
+        @parser.cleaned_row_hashes.first.should == pujols_row_hash
+        # @parser.cleaned_row_hashes.last.should == {:table => 'cities', :values => {'name' => 'Madison'}}
+      end
+    end
+    
+    describe 'howard' do
+      before do
+        @parser = TableParser.new(:filename => spec_file('howard.html'), :table => 'howard')
+      end
+      it 'row count' do
+        @parser.value_hashes.should size(6)
+      end
+      it 'value hash' do
+        @parser.cleaned_value_hashes[1]['PTS'].should == 15.8
+      end
+    end
+  end
+  
+end

data/spec/howard.html

@@ -0,0 +1,7 @@
+YR	TM	G	GS	MIN	FG	FG%	3P	3P%	FT	FT%	STL	BLK	TO	PF	OFF	DEF	TOT	AST	PTS
+04-05	ORL	82	82	32.6	4.3-8.3	.520	0.0-0.0	.000	3.4-5.0	.671	0.90	1.70	2.0	2.8	3.5	6.5	10.0	0.9	12.0
+05-06	ORL	82	81	36.8	5.7-10.7	.531	0.0-0.0	.000	4.3-7.3	.595	0.80	1.40	2.6	3.4	3.5	9.0	12.5	1.5	15.8
+06-07	ORL	82	82	36.9	6.4-10.6	.603	0.0-0.0	.500	4.8-8.1	.586	0.90	1.90	3.9	3.0	3.5	8.8	12.3	1.9	17.6
+07-08	ORL	82	82	37.7	7.1-11.9	.599	0.0-0.0	.000	6.5-10.9	.590	0.90	2.10	3.2	3.3	3.4	10.8	14.2	1.3	20.7
+08-09	ORL	79	79	35.7	7.1-12.4	.572	0.0-0.0	.000	6.4-10.7	.594	1.00	2.90	3.0	3.4	4.3	9.6	13.9	1.4	20.6
+09-10	ORL	82	82	34.7	6.2-10.2	.612	0.0-0.1	.000	5.9-10.0	.592	0.90	2.80	3.3	3.5	3.5	9.7	13.2	1.8	18.3

data/spec/junk.rb

@@ -0,0 +1,24 @@
+class MyProxyFoo
+  attr_accessor :obj_block
+  include FromHash
+  fattr(:obj) { obj_block[] }
+  def method_missing(sym,*args,&b)
+   # raise "sending #{args.inspect}"
+    obj.send(sym,*args,&b)
+  end
+end
+
+def try_all(*args)
+  args.flatten.each do |arg|
+    begin
+      yield(arg)
+      puts "#{arg} worked"
+      return
+    rescue(RuntimeError)
+      puts "runtimeerror"
+    rescue
+      puts "generic error"
+    end
+  end
+  raise "none worked"
+end

data/spec/players.csv

@@ -0,0 +1,4 @@
+first,last,age
+Albert,Pujols,29
+David,Wright,26
+Hanley,Ramirez,27

data/spec/players.sql

@@ -0,0 +1,13 @@
+select * from foo
+CREATE TABLE PLAYERS (
+insert stuff
+)
+#INSERT into players (first,last,age) VALUES ("Hanley", 'Ramirez',27);    
+    
+INSERT into players
+(first,last,age) 
+VALUES ("Albert", 'Pujols',29)
+
+INSERT into players (first,last,age) VALUES ("Hanley", 'Ramirez',27);    
+    
+INSERT into players (first,last,age) VALUES ("David", 'Wright',26);       

data/spec/players.xml

@@ -0,0 +1,24 @@
+<top>
+  <players>
+    <player>
+      <first>Albert</first>
+      <last>Pujols</last>
+      <age>29</age>
+    </player>
+    <player>
+      <first>David</first>
+      <last>Wright</last>
+      <age>26</age>
+    </player>
+    <player>
+      <first>Hanley</first>
+      <last>Ramirez</last>
+      <age>27</age>
+    </player>
+  </players>
+  <cities>
+    <city>
+      <name>Madison</name>
+    </city>
+  </cities>
+</top>

data/spec/preview_spec.rb

@@ -0,0 +1,49 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "ImportEverything" do
+  describe Preview do
+    describe 'csv' do
+      before do
+        @preview = Preview.new(:filename => spec_file('players.csv'), :table => 'players')
+      end
+      it 'ready' do
+        @preview.should be_ready
+      end
+      it 'values' do
+        table = @preview.preview_table
+        table.row_value_arrays.first.should == ['Albert','Pujols',29]
+        table.row_value_arrays.should size(3)
+      end
+      it 'keys' do
+        @preview.preview_table.keys.should == ['first','last','age']
+      end
+    end
+    
+    describe 'xml' do
+      before do
+        @preview = Preview.new(:filename => spec_file('players.xml'))
+        #@preview.table_paths = nil
+      end
+      it 'should need fields' do
+        @preview.addl_required_fields.should == [:root_path,:table_paths]
+      end
+      it 'can set needed fields' do
+        @preview.root_path = 'abc'
+        @preview.addl_required_fields.should == [:table_paths]
+      end
+      describe 'ready' do
+        before do
+          @preview.root_path = 'top'
+          @preview.table_paths = [['players','player']]
+        end
+        it 'ready' do
+          @preview.should be_ready
+        end
+        it 'values' do
+          @preview.table.row_value_arrays.first.should == ['Albert','Pujols',29]
+        end
+      end
+    end
+  end
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,39 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'import_everything'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end
+
+include ImportEverything
+
+def mit(name,&b)
+  it(name,&b) #if name == 'parses values'
+end
+
+Spec::Matchers.define :size do |exp_size|
+  match do |arr|
+    arr.size == exp_size
+  end
+  failure_message_for_should do |arr|
+    "Array is size #{arr.size}, expected #{exp_size}, array is #{arr.inspect}"
+  end
+end
+
+def pujols_value_hash
+  {'first' => 'Albert', 'last' => 'Pujols', 'age' => 29}
+end
+
+def pujols_row_hash
+  {:table => 'players', :values => pujols_value_hash}
+end
+
+def spec_file(f)
+  File.expand_path(File.dirname(__FILE__)) + "/#{f}"
+end
+
+def mylog(*args)
+end

data/spec/sql_parser_spec.rb

@@ -0,0 +1,106 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "ImportEverything" do
+  describe SqlInsertParser do
+    before do
+      @pujols_value_hash = pujols_value_hash
+    end
+    describe SqlInsertParser::LineParser do
+      before do
+        @sql = "INSERT into players (first ,LAST,age) VALUES (\"Albert\", 'Pujols',29)"
+      end
+      fattr(:parser) do
+        SqlInsertParser::LineParser.new(:line => @sql)
+      end
+      SqlInsertParser::LineParser.regex_hash.each do |name,reg|
+        mit "can match against #{name} regex" do
+          (@sql =~ reg).should be
+        end
+      end
+      mit 'parses table name' do
+        parser.table.should == 'players'
+      end
+      mit 'parses raw columns' do
+        parser.parsed_elements[:columns].should == 'first ,LAST,age'
+      end
+      mit 'parses raw values' do
+        parser.parsed_elements[:values].should == "\"Albert\", 'Pujols',29"
+      end
+      mit 'parses columns' do
+        parser.columns.should == %w(first last age)
+      end
+      mit 'parses values' do
+        parser.values.should == ['Albert','Pujols',29]
+      end
+      mit 'value hash' do
+        parser.value_hash.should == @pujols_value_hash
+      end
+      mit 'quote regex match' do
+        ('"abc"' =~ /^['"].*['"]$/).should be
+        '"abc"'.should =~ /^['"].*['"]$/
+        "'abc'".should =~ /^['"].*['"]$/
+      end
+      it "ignore commented lines" do
+        line = "# INSERT INTO players (a,b,c) VALUES (d,e,f)"
+        SqlInsertParser::LineParser.should_not be_probable_insert(line)
+      end
+      it 'can handle quoted table' do
+        @sql = "INSERT into 'players' (first ,LAST,'age') VALUES (\"Albert\", 'Pujols',29)"
+        parser.table.should == 'players'
+      end
+      it 'can handle quoted column' do
+        @sql = "INSERT into players (first ,LAST,'age') VALUES (\"Albert\", 'Pujols',29)"
+        parser.columns.should == %w(first last age)
+      end
+      it 'can handle multi line insert' do
+        @sql = "INSERT into players (first,last,age) 
+                VALUES (\"Albert\", 'Pujols',29)"
+        parser.values.should == ['Albert','Pujols',29]
+        parser.value_hash.should == @pujols_value_hash
+      end
+    end
+    
+    describe 'file parsing' do
+      before do
+        @lines = ["select * from foo","CREATE TABLE PLAYERS (","insert stuff",")"]
+        @lines << "#INSERT into players (first,last,age) VALUES (\"Hanley\", 'Ramirez',27);    \n    "
+        @lines << "INSERT into players (first,last,age) VALUES (\"Albert\", 'Pujols',29)\n"
+        @lines << "INSERT into players (first,last,age) VALUES (\"Hanley\", 'Ramirez',27);    \n    "
+        @lines << "INSERT into players (first,last,age) VALUES (\"David\", 'Wright',26);       "
+      end
+      fattr(:raw_sql) { @lines.join("\n") }
+      fattr(:sql) do
+        File.create(spec_file("players.sql"),raw_sql)
+        raw_sql
+      end
+      fattr(:parser) { SqlInsertParser.new(:str => sql) }
+      it 'has right number of lines' do
+        parser.lines.size.should == @lines.size
+      end
+      it 'has 3 probable lines' do
+        parser.probable_insert_lines.size.should == 3
+      end
+      it 'value hashes' do
+        #raise parser.insert_lines.inspect
+        parser.value_hashes.size.should == 3
+        parser.value_hashes.first.should == @pujols_value_hash
+        parser.value_hashes[1].should == {'first' => 'Hanley', 'last' => 'Ramirez', 'age' => 27}
+      end
+      it 'row hashes' do
+        parser.row_hashes.first.should == pujols_row_hash
+      end
+      describe 'multi line insert' do
+        before do
+          @lines[-3] = @lines[-3].gsub(/VALUES/,"\nVALUES").gsub(/players /,"players\n")
+        end
+        it 'right number of insert lines' do
+          parser.insert_lines.should size(3)
+        end
+        it 'right value hash' do
+          parser.value_hashes.first.should == @pujols_value_hash
+        end
+      end
+    end
+  end
+  
+end

data/spec/sqlite_parser_spec.rb

@@ -0,0 +1,56 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+class TestDB
+  class << self
+    fattr(:instance) { new }
+  end
+  fattr(:filename) { "/Code/wheeeee.sqlite3" }
+  fattr(:db) do
+    gem 'sqlite3-ruby'
+    require 'sqlite3'
+    SQLite3::Database.new(filename) 
+    #SQLite3::Database.new(open("http://localhost:3000/wheeeee.sqlite3") { |f| f.read })
+  end
+  fattr(:crefate) do
+    `rm -f #{filename}` 
+    db.execute("CREATE TABLE cities ( name varchar(255) )")
+    db.execute("CREATE TABLE players ( first varchar(255), last varchar(255), age integer)")
+    #raise db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;").inspect
+    db.execute("INSERT into players (first,last,age) VALUES ('Albert','Pujols',29)")
+    db.execute("INSERT into players (first,last,age) VALUES ('David','Wright',26)")
+    db.execute("INSERT into players (first,last,age) VALUES ('Hanley','Ramirez',27)")
+    db.execute("INSERT into cities (name) VALUES ('Madison')")
+    raise "foo" unless db.execute("select count(*) from players").flatten.first.to_i == 3
+  end
+  def create; end
+end
+def create_test_db!
+  
+end
+
+# raise TestDB.instance.db.execute2("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;").flatten.inspect
+
+describe "ImportEverything" do
+  describe SqliteParser do
+    before do
+      TestDB.instance.create
+      #@parser = SqliteParser.new(:filename => TestDB.instance.filename)
+      @parser = ImportEverything.get_parser(:filename => TestDB.instance.filename)
+      @rows = ImportEverything.get_rows(:filename => TestDB.instance.filename)
+    end
+    it 'tables' do
+      @parser.tables.should == ['cities','players']
+    end
+    it 'value hashes' do
+      @parser.cleaned_value_hashes.size.should == 4
+      @parser.cleaned_value_hashes[1].should == pujols_value_hash
+    end
+    it 'row hashes' do
+      @parser.cleaned_row_hashes.first.should == {:table => 'cities', :values => {'name' => 'Madison'}}
+      @parser.cleaned_row_hashes[1].should == pujols_row_hash
+      @rows.first.should == {:table => 'cities', :values => {'name' => 'Madison'}}
+      @rows[1].should == pujols_row_hash
+    end
+  end
+  
+end

data/spec/xml_parser_spec.rb

@@ -0,0 +1,31 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "ImportEverything" do
+  describe XmlParser do
+    before do
+      @parser = XmlParser.new(:filename => "/Code/import_everything/spec/players.xml", :root_path => 'top', :table_paths => [['players','player'],['cities','city']])
+      #@parser.table_paths = nil
+    end
+    it 'smoke' do
+      2.should == 2
+    end
+    it 'parsers' do
+      @parser.parsers
+    end
+    it 'row count' do
+      @parser.value_hashes.should size(4)
+      @parser.cleaned_row_hashes.should size(4)
+    end
+    it 'first doc' do
+      @parser.parsers.first.cleaned_value_hashes.first.should == pujols_value_hash
+    end
+    it 'hashes' do
+      @parser.parsers.first.cleaned_value_hashes
+    end
+    it 'city' do
+      @parser.cleaned_row_hashes.first.should == pujols_row_hash
+      @parser.cleaned_row_hashes.last.should == {:table => 'cities', :values => {'name' => 'Madison'}}
+    end
+  end
+  
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification 
+name: import_everything
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Mike Harris
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-18 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: import everything
+email: mharris717@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/import_everything.rb
+- lib/import_everything/ext.rb
+- lib/import_everything/parsers/csv_parser.rb
+- lib/import_everything/parsers/determine_type.rb
+- lib/import_everything/parsers/line_parser.rb
+- lib/import_everything/parsers/module_methods.rb
+- lib/import_everything/parsers/parser.rb
+- lib/import_everything/parsers/preview.rb
+- lib/import_everything/parsers/sql_parser.rb
+- lib/import_everything/parsers/sqlite_parser.rb
+- lib/import_everything/parsers/table_parser.rb
+- lib/import_everything/parsers/xml_parser.rb
+- spec/bets.html
+- spec/csv_parser_spec.rb
+- spec/howard.html
+- spec/import_everything_spec.rb
+- spec/junk.rb
+- spec/players.csv
+- spec/players.sql
+- spec/players.xml
+- spec/preview_spec.rb
+- spec/spec_helper.rb
+- spec/sql_parser_spec.rb
+- spec/sqlite_parser_spec.rb
+- spec/xml_parser_spec.rb
+has_rdoc: true
+homepage: http://github.com/mharris717/import_everything
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: import everything
+test_files: 
+- spec/csv_parser_spec.rb
+- spec/import_everything_spec.rb
+- spec/junk.rb
+- spec/preview_spec.rb
+- spec/spec_helper.rb
+- spec/sql_parser_spec.rb
+- spec/sqlite_parser_spec.rb
+- spec/xml_parser_spec.rb