import_everything 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.document ADDED
@@ -0,0 +1,5 @@
1
+ README.rdoc
2
+ lib/**/*.rb
3
+ bin/*
4
+ features/**/*.feature
5
+ LICENSE
data/.gitignore ADDED
@@ -0,0 +1,5 @@
1
+ *.sw?
2
+ .DS_Store
3
+ coverage
4
+ rdoc
5
+ pkg
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2009 Mike Harris
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.rdoc ADDED
@@ -0,0 +1,18 @@
1
+ = import_everything
2
+
3
+ Description goes here.
4
+
5
+ == Note on Patches/Pull Requests
6
+
7
+ * Fork the project.
8
+ * Make your feature addition or bug fix.
9
+ * Add tests for it. This is important so I don't break it in a
10
+ future version unintentionally.
11
+ * Commit, do not mess with rakefile, version, or history.
12
+ (if you want to have your own version, that is fine but
13
+ bump version in a commit by itself I can ignore when I pull)
14
+ * Send me a pull request. Bonus points for topic branches.
15
+
16
+ == Copyright
17
+
18
+ Copyright (c) 2010 Mike Harris. See LICENSE for details.
data/Rakefile ADDED
@@ -0,0 +1,49 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+
4
+ begin
5
+ require 'jeweler'
6
+ Jeweler::Tasks.new do |gem|
7
+ gem.name = "import_everything"
8
+ gem.summary = %Q{import everything}
9
+ gem.description = %Q{import everything}
10
+ gem.email = "mharris717@gmail.com"
11
+ gem.homepage = "http://github.com/mharris717/import_everything"
12
+ gem.authors = ["Mike Harris"]
13
+ gem.add_development_dependency "rspec"
14
+ # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
15
+ end
16
+ Jeweler::GemcutterTasks.new
17
+ rescue LoadError
18
+ puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
19
+ end
20
+
21
+ require 'spec/rake/spectask'
22
+ Spec::Rake::SpecTask.new(:spec) do |spec|
23
+ spec.libs << 'lib' << 'spec'
24
+ spec.spec_files = FileList['spec/**/*_spec.rb']
25
+ end
26
+
27
+ Spec::Rake::SpecTask.new(:rcov) do |spec|
28
+ spec.libs << 'lib' << 'spec'
29
+ spec.pattern = 'spec/**/*_spec.rb'
30
+ spec.rcov = true
31
+ end
32
+
33
+ task :spec => :check_dependencies
34
+
35
+ task :default => :spec
36
+
37
+ require 'rake/rdoctask'
38
+ Rake::RDocTask.new do |rdoc|
39
+ if File.exist?('VERSION')
40
+ version = File.read('VERSION')
41
+ else
42
+ version = ""
43
+ end
44
+
45
+ rdoc.rdoc_dir = 'rdoc'
46
+ rdoc.title = "import_everything #{version}"
47
+ rdoc.rdoc_files.include('README*')
48
+ rdoc.rdoc_files.include('lib/**/*.rb')
49
+ end
data/VERSION ADDED
@@ -0,0 +1 @@
1
+ 0.1.0
@@ -0,0 +1,92 @@
1
+ class Object
2
+ def local_methods
3
+ res = methods - 7.methods - ''.methods - Object.new.methods
4
+ res.sort
5
+ end
6
+ end
7
+
8
+ class String
9
+ def blank?
10
+ strip == ''
11
+ end
12
+ def present?
13
+ !blank?
14
+ end
15
+ end
16
+
17
+ module MethodLogging
18
+ def log_method(name)
19
+ alias_method "unlogged_#{name}", name
20
+ define_method(name) do |*args|
21
+ res = send("unlogged_#{name}",*args)
22
+ puts "#{name} returned #{res.inspect} from #{args.inspect}"
23
+ res
24
+ end
25
+ end
26
+ def def_logging_method(name,&b)
27
+ define_method(name) do |*args|
28
+ res = instance_eval()
29
+ end
30
+ end
31
+ end
32
+
33
+ class Hash
34
+ def self.from_keys_and_values(ks,vs)
35
+ raise "size not equal #{ks.size} #{vs.size}" unless ks.size == vs.size
36
+ ks.zip(vs).inject({}) { |h,a| h.merge(a[0] => a[1]) }
37
+ end
38
+ end
39
+
40
+ class String
41
+ def without_quotes
42
+ if self =~ /^['"].*['"]$/
43
+ self[1..-2]
44
+ else
45
+ self
46
+ end
47
+ end
48
+ def to_num_if
49
+ if self =~ /^\d+$/
50
+ self.to_i
51
+ elsif self =~ /^[\d\.]+$/
52
+ self.to_f
53
+ else
54
+ self
55
+ end
56
+ end
57
+ def fixed_obj
58
+ strip.without_quotes.to_num_if
59
+ end
60
+ end
61
+
62
+ class Object
63
+ def fixed_obj
64
+ self
65
+ end
66
+ end
67
+
68
+ module Enumerable
69
+ def make_last
70
+ h = group_by { |x| !!yield(x) }
71
+ (h[false]||[]) + (h[true]||[])
72
+ end
73
+ def make_first
74
+ h = group_by { |x| !!yield(x) }
75
+ (h[true]||[]) + (h[false]||[])
76
+ end
77
+ end
78
+
79
+ class Hash
80
+ def cleaned_hash_values
81
+ map_value { |v| v.fixed_obj }
82
+ end
83
+ end
84
+
85
+ class Object
86
+ def first_responding(*methods)
87
+ methods.flatten.each { |x| return send(x) if respond_to?(x) }
88
+ #raise "none respond"
89
+ #raise local_methods.inspect
90
+ nil
91
+ end
92
+ end
@@ -0,0 +1,16 @@
1
+ module ImportEverything
2
+ class CsvParser < Parser
3
+ fattr(:delimiter) { "," }
4
+ def value_hashes
5
+ require 'csv'
6
+ res = []
7
+ CSV.parse(str, :headers => true, :col_sep => delimiter) do |row|
8
+ res << row.to_hash
9
+ end
10
+ res
11
+ end
12
+ def required_fields
13
+ []
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,36 @@
1
+ module ImportEverything
2
+ class DetermineType
3
+ include FromHash
4
+ def self.get_filename(file)
5
+ if file.kind_of?(String)
6
+ ''
7
+ else
8
+ file.first_responding(:original_filename, :filename, :path, :name)
9
+ end
10
+ end
11
+ fattr(:filename) { self.class.get_filename(file) }
12
+ fattr(:file) { open(filename) }
13
+ fattr(:ext) { filename.split(".").last.downcase }
14
+ def method_missing(sym,*args,&b)
15
+ if sym.to_s[-1..-1] == '='
16
+ self.addl_ops[sym.to_s[0..-2]] = args.first
17
+ else
18
+ super
19
+ end
20
+ end
21
+ fattr(:addl_ops) { {} }
22
+ fattr(:parser_ops) { {:file => file}.merge(addl_ops) }
23
+ def parser_class
24
+ h = {'sqlite' => SqliteParser, 'sqlite3' => SqliteParser, 'csv' => CsvParser, 'xml' => XmlParser, 'sql' => SqlInsertParser, 'dmp' => SqlInsertParser, 'html' => TableParser}
25
+ h[ext].tap { |x| return x if x }
26
+ h.each do |k,klass|
27
+ return klass if ext =~ /^#{k}\d\d/
28
+ end
29
+ raise "no parser found for #{ext}"
30
+ end
31
+ def parser
32
+ # puts "parser ops is #{parser_ops.inspect}"
33
+ parser_class.new(parser_ops)
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,13 @@
1
+ module ImportEverything
2
+ class LineParser
3
+ def value_hashes
4
+ [value_hash]
5
+ end
6
+ fattr(:cleaned_value_hashes) do
7
+ value_hashes.map { |row| row.cleaned_hash_values }
8
+ end
9
+ def row_hashes
10
+ value_hashes.map { |x| {:table => table, :values => x} }
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,18 @@
1
+ module ImportEverything
2
+ def self.get_parser(ops)
3
+ dt = ImportEverything::DetermineType.new(ops)
4
+ dt.parser
5
+ end
6
+ def self.get_rows(ops)
7
+ get_parser(ops).cleaned_row_hashes
8
+ end
9
+ def self.each_row(ops,&b)
10
+ get_parser(ops).each_row(&b)
11
+ end
12
+ def self.each_table_and_rows(ops,&b)
13
+ get_parser(ops).each_table_and_rows(&b)
14
+ end
15
+ def self.preview(ops)
16
+ Preview.new(ops)
17
+ end
18
+ end
@@ -0,0 +1,55 @@
1
+ module ImportEverything
2
+ class Parser
3
+ fattr(:table) { 'some_table' }
4
+ include FromHash
5
+ fattr(:cleaned_value_hashes) do
6
+ value_hashes.map { |row| row.cleaned_hash_values }
7
+ end
8
+ fattr(:cleaned_row_hashes) do
9
+ row_hashes.map do |row_hash|
10
+ {:table => row_hash[:table], :values => row_hash[:values].cleaned_hash_values}
11
+ end
12
+ end
13
+
14
+ fattr(:value_hashes) do
15
+ parsers.map { |x| x.value_hashes }.flatten
16
+ end
17
+ fattr(:row_hashes) do
18
+ if respond_to?(:parsers)
19
+ parsers.map { |x| x.row_hashes }.flatten
20
+ else
21
+ value_hashes.map { |x| {:table => table, :values => x} }
22
+ end
23
+ end
24
+ def line_parsers; parsers; end
25
+ fattr(:filename) { ImportEverything::DetermineType.get_filename(file) }
26
+ fattr(:file) { open(filename) }
27
+ fattr(:str) { file.read }
28
+
29
+ def each_row
30
+ each_table_and_rows do |table,rows|
31
+ rows.each { |row| yield(table,row) }
32
+ end
33
+ end
34
+ def each_table_and_rows
35
+ cleaned_row_hashes.group_by { |x| x[:table] }.each do |table,rows|
36
+ values = rows.map { |x| x[:values] }
37
+ yield(table,values)
38
+ end
39
+ end
40
+ def table_rows_hash
41
+ res = {}
42
+ each_table_and_rows { |table,rows| res[table] = rows }
43
+ res
44
+ end
45
+ end
46
+ module ParserPreviewMod
47
+ fattr(:addl_required_fields) do
48
+ required_fields.select do |x|
49
+ send(x).to_s.blank?
50
+ end
51
+ end
52
+ fattr(:required_fields) { [] }
53
+ end
54
+ Parser.send(:include,ParserPreviewMod)
55
+ end
@@ -0,0 +1,58 @@
1
+ module ImportEverything
2
+ class DisplayTable
3
+ attr_accessor :rows, :table
4
+ include FromHash
5
+ fattr(:keys) do
6
+ rows.map { |x| x.keys }.flatten.uniq
7
+ end
8
+ def each_row_value_array
9
+ rows.each do |row|
10
+ vals = keys.map { |k| row[k] }
11
+ vals.each { |x| yield(x) }
12
+ end
13
+ end
14
+ def row_value_arrays
15
+ rows.map do |row|
16
+ keys.map { |k| row[k] }
17
+ end
18
+ end
19
+ def rows
20
+ #mylog "display_table", :rows => @rows
21
+ @rows
22
+ end
23
+ def to_hash
24
+ {:keys => keys, :row_arrays => row_value_arrays, :table => table}
25
+ end
26
+ end
27
+ class Preview
28
+ attr_accessor :ops
29
+ def initialize(ops)
30
+ self.ops = ops
31
+ end
32
+ fattr(:dt) { ImportEverything::DetermineType.new(ops) }
33
+ def parser
34
+ dt.parser
35
+ end
36
+ def addl_required_fields
37
+ parser.addl_required_fields
38
+ end
39
+ def ready?
40
+ addl_required_fields.empty?
41
+ end
42
+ def preview_table
43
+ DisplayTable.new(:rows => parser.cleaned_value_hashes)
44
+ end
45
+ def table; preview_table; end
46
+ def tables
47
+ parser.table_rows_hash.map { |table,rows| DisplayTable.new(:table => table, :rows => rows) }
48
+ end
49
+ def method_missing(sym,*args,&b)
50
+ if sym.to_s[-1..-1] == '='
51
+ self.ops[sym.to_s[0..-2]] = args.first
52
+ dt!
53
+ else
54
+ super
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,79 @@
1
+ module ImportEverything
2
+ class SqlInsertParser < Parser
3
+ fattr(:lines) do
4
+ str.split(/\n|;/).map { |x| x.strip }.select { |x| x.present? }
5
+ end
6
+ fattr(:probable_insert_lines) do
7
+ lines.select { |x| LineParser.probable_insert?(x) }
8
+ end
9
+ def get_insert_lines(lines)
10
+ if lines.empty?
11
+ []
12
+ elsif LineParser.comment?(lines.first)
13
+ get_insert_lines(lines[1..-1])
14
+ elsif LineParser.valid_insert?(lines.first)
15
+ [lines.first] + get_insert_lines(lines[1..-1])
16
+ elsif lines.size >= 2 && LineParser.valid_insert?(lines[0..1].join(" "))
17
+ [lines[0..1].join(" ")] + get_insert_lines(lines[2..-1])
18
+ elsif lines.size >= 3 && LineParser.valid_insert?(lines[0..2].join(" "))
19
+ [lines[0..2].join(" ")] + get_insert_lines(lines[2..-1])
20
+ else
21
+ get_insert_lines(lines[1..-1])
22
+ end
23
+ end
24
+ fattr(:insert_lines) do
25
+ get_insert_lines(lines)
26
+ end
27
+ fattr(:parsers) do
28
+ insert_lines.map { |ln| LineParser.new(:line => ln) }
29
+ end
30
+
31
+ class LineParser < LineParser
32
+ attr_accessor :line
33
+ include FromHash
34
+ extend MethodLogging
35
+ class << self
36
+ fattr(:insert_into_regex) { /INSERT\s+INTO\s+(\S+)/i }
37
+ fattr(:columns_regex) { /\((.*)\)/i }
38
+ fattr(:values_regex) { /VALUES\s+\((.*)\)/i }
39
+ fattr(:full_regex) { /^\s*#{insert_into_regex}\s+#{columns_regex}\s+#{values_regex}/i }
40
+ fattr(:regex_hash) do
41
+ {:insert => insert_into_regex, :columns => columns_regex, :values => values_regex, :full => full_regex}
42
+ end
43
+ fattr(:comment_regex) do
44
+ /^(\/\*|#)/
45
+ end
46
+ def probable_insert?(str)
47
+ str = str.strip
48
+ res = (str =~ insert_into_regex) && !comment?(str)
49
+ !!res
50
+ end
51
+ def valid_insert?(str)
52
+ !!(str =~ full_regex)
53
+ end
54
+ def comment?(str)
55
+ !!(str =~ comment_regex)
56
+ end
57
+ end
58
+ fattr(:parsed_elements) do
59
+ raise "doesn't parse" unless line =~ self.class.full_regex
60
+ {:table => $1, :columns => $2, :values => $3}
61
+ end
62
+ fattr(:table) { parsed_elements[:table].without_quotes }
63
+ fattr(:columns) do
64
+ parsed_elements[:columns].split(",").map { |x| x.downcase.strip.without_quotes }
65
+ end
66
+ #log_method :fix_value
67
+ fattr(:values) do
68
+ require 'csv'
69
+ raw = parsed_elements[:values]
70
+ CSV.parse(raw).first.map { |x| x.fixed_obj }
71
+ end
72
+ fattr(:value_hash) do
73
+ Hash.from_keys_and_values(columns,values)
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+
@@ -0,0 +1,46 @@
1
+ module ImportEverything
2
+ class SqliteParser < Parser
3
+ fattr(:filfename) do
4
+ f = Tempfile.new('somedb.sqlite3')
5
+ f.binmode
6
+ f << file.read
7
+ f.path
8
+ end
9
+ fattr(:db) do
10
+ gem 'sqlite3-ruby'
11
+ require 'sqlite3'
12
+ #raise "#{filename} #{file.local_methods.inspect}"
13
+ res = SQLite3::Database.new(filename)
14
+ #get_raw_tables(res)
15
+ #res
16
+ end
17
+ def get_raw_tables(db)
18
+ sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
19
+ res = db.execute(sql).flatten
20
+ mylog "sqlite", :res => res, :sql => sql
21
+ res
22
+ end
23
+ fattr(:raw_tables) { get_raw_tables(db) }
24
+ fattr(:tables) do
25
+ raw_tables.reject { |x| x =~ /^(schema|sqlite)/ }
26
+ end
27
+ fattr(:parsers) do
28
+ tables.map { |x| TableParser.new(:db => db, :table => x) }
29
+ end
30
+ def required_fields
31
+ []
32
+ end
33
+ class TableParser < LineParser
34
+ attr_accessor :db, :table
35
+ include FromHash
36
+ fattr(:execute_result) { db.execute2("SELECT * FROM #{table}") }
37
+ fattr(:columns) { execute_result.first }
38
+ fattr(:raw_rows) { execute_result[1..-1] }
39
+ fattr(:value_hashes) do
40
+ raw_rows.map do |row|
41
+ Hash.from_keys_and_values(columns,row)
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,5 @@
1
+ module ImportEverything
2
+ class TableParser < CsvParser
3
+ fattr(:delimiter) { "\t" }
4
+ end
5
+ end
@@ -0,0 +1,38 @@
1
+ module ImportEverything
2
+ class XmlParser < Parser
3
+ attr_accessor :root_path, :table_paths, :xyz
4
+ #fattr(:table_paths) { [['players','player'],['cities','city']] }
5
+ #fattr(:root_path) { 'top' }
6
+ fattr(:raw_doc) { require 'hpricot'; Hpricot(str) }
7
+ fattr(:doc) { raw_doc/root_path }
8
+ fattr(:parsers) do
9
+ table_paths.map do |table_desc|
10
+ docs = (doc/table_desc[0])
11
+ TableParser.new(:table => table_desc[0], :doc => docs, :row_path => table_desc[1])
12
+ end
13
+ end
14
+ def required_fields
15
+ [:root_path,:table_paths]
16
+ end
17
+
18
+ class TableParser < Parser
19
+ attr_accessor :doc, :table, :row_path
20
+ include FromHash
21
+ def row_docs
22
+ doc/row_path
23
+ end
24
+ def parsers
25
+ row_docs.map { |doc| LineParser.new(:doc => doc, :table => table) }
26
+ end
27
+ end
28
+
29
+ class LineParser < LineParser
30
+ attr_accessor :doc, :table
31
+ include FromHash
32
+ def value_hash
33
+ cs = doc.children.reject { |x| x.to_s.blank? }
34
+ cs.inject({}) { |h,el| h.merge(el.name => el.inner_text) }
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,9 @@
1
+ require 'rubygems'
2
+ require 'mharris_ext'
3
+ require 'active_support'
4
+
5
+ this_path = File.expand_path(File.dirname(__FILE__))
6
+ require "#{this_path}/import_everything/ext"
7
+ paths = Dir["#{this_path}/import_everything/**/*.rb"]
8
+ paths = paths.make_first { |path| %w(parser.rb line_parser.rb).include?(File.basename(path)) }
9
+ paths.each { |x| require x }
data/spec/bets.html ADDED
@@ -0,0 +1,3 @@
1
+ Event Name Market Type Selection Odds Stake ($) Market P&L ($)
2
+ Eastern Conference Winner Orlando (to win) +282 157.09 443.00
3
+ Carib Stupidity Carib Stupidity Vegas Watch 107494 (to win) +180 1.00 1.79
@@ -0,0 +1,35 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "ImportEverything" do
4
+ describe CsvParser do
5
+ describe 'players' do
6
+ before do
7
+ @parser = CsvParser.new(:filename => spec_file('players.csv'), :table => 'players')
8
+ end
9
+ it 'smoke' do
10
+ 2.should == 2
11
+ end
12
+ it 'row count' do
13
+ @parser.value_hashes.should size(3)
14
+ @parser.cleaned_row_hashes.should size(3)
15
+ end
16
+ it 'cleaned hash' do
17
+ @parser.cleaned_row_hashes.first.should == pujols_row_hash
18
+ # @parser.cleaned_row_hashes.last.should == {:table => 'cities', :values => {'name' => 'Madison'}}
19
+ end
20
+ end
21
+
22
+ describe 'howard' do
23
+ before do
24
+ @parser = TableParser.new(:filename => spec_file('howard.html'), :table => 'howard')
25
+ end
26
+ it 'row count' do
27
+ @parser.value_hashes.should size(6)
28
+ end
29
+ it 'value hash' do
30
+ @parser.cleaned_value_hashes[1]['PTS'].should == 15.8
31
+ end
32
+ end
33
+ end
34
+
35
+ end
data/spec/howard.html ADDED
@@ -0,0 +1,7 @@
1
+ YR TM G GS MIN FG FG% 3P 3P% FT FT% STL BLK TO PF OFF DEF TOT AST PTS
2
+ 04-05 ORL 82 82 32.6 4.3-8.3 .520 0.0-0.0 .000 3.4-5.0 .671 0.90 1.70 2.0 2.8 3.5 6.5 10.0 0.9 12.0
3
+ 05-06 ORL 82 81 36.8 5.7-10.7 .531 0.0-0.0 .000 4.3-7.3 .595 0.80 1.40 2.6 3.4 3.5 9.0 12.5 1.5 15.8
4
+ 06-07 ORL 82 82 36.9 6.4-10.6 .603 0.0-0.0 .500 4.8-8.1 .586 0.90 1.90 3.9 3.0 3.5 8.8 12.3 1.9 17.6
5
+ 07-08 ORL 82 82 37.7 7.1-11.9 .599 0.0-0.0 .000 6.5-10.9 .590 0.90 2.10 3.2 3.3 3.4 10.8 14.2 1.3 20.7
6
+ 08-09 ORL 79 79 35.7 7.1-12.4 .572 0.0-0.0 .000 6.4-10.7 .594 1.00 2.90 3.0 3.4 4.3 9.6 13.9 1.4 20.6
7
+ 09-10 ORL 82 82 34.7 6.2-10.2 .612 0.0-0.1 .000 5.9-10.0 .592 0.90 2.80 3.3 3.5 3.5 9.7 13.2 1.8 18.3
File without changes
data/spec/junk.rb ADDED
@@ -0,0 +1,24 @@
1
+ class MyProxyFoo
2
+ attr_accessor :obj_block
3
+ include FromHash
4
+ fattr(:obj) { obj_block[] }
5
+ def method_missing(sym,*args,&b)
6
+ # raise "sending #{args.inspect}"
7
+ obj.send(sym,*args,&b)
8
+ end
9
+ end
10
+
11
+ def try_all(*args)
12
+ args.flatten.each do |arg|
13
+ begin
14
+ yield(arg)
15
+ puts "#{arg} worked"
16
+ return
17
+ rescue(RuntimeError)
18
+ puts "runtimeerror"
19
+ rescue
20
+ puts "generic error"
21
+ end
22
+ end
23
+ raise "none worked"
24
+ end
data/spec/players.csv ADDED
@@ -0,0 +1,4 @@
1
+ first,last,age
2
+ Albert,Pujols,29
3
+ David,Wright,26
4
+ Hanley,Ramirez,27
data/spec/players.sql ADDED
@@ -0,0 +1,13 @@
1
+ select * from foo
2
+ CREATE TABLE PLAYERS (
3
+ insert stuff
4
+ )
5
+ #INSERT into players (first,last,age) VALUES ("Hanley", 'Ramirez',27);
6
+
7
+ INSERT into players
8
+ (first,last,age)
9
+ VALUES ("Albert", 'Pujols',29)
10
+
11
+ INSERT into players (first,last,age) VALUES ("Hanley", 'Ramirez',27);
12
+
13
+ INSERT into players (first,last,age) VALUES ("David", 'Wright',26);
data/spec/players.xml ADDED
@@ -0,0 +1,24 @@
1
+ <top>
2
+ <players>
3
+ <player>
4
+ <first>Albert</first>
5
+ <last>Pujols</last>
6
+ <age>29</age>
7
+ </player>
8
+ <player>
9
+ <first>David</first>
10
+ <last>Wright</last>
11
+ <age>26</age>
12
+ </player>
13
+ <player>
14
+ <first>Hanley</first>
15
+ <last>Ramirez</last>
16
+ <age>27</age>
17
+ </player>
18
+ </players>
19
+ <cities>
20
+ <city>
21
+ <name>Madison</name>
22
+ </city>
23
+ </cities>
24
+ </top>
@@ -0,0 +1,49 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "ImportEverything" do
4
+ describe Preview do
5
+ describe 'csv' do
6
+ before do
7
+ @preview = Preview.new(:filename => spec_file('players.csv'), :table => 'players')
8
+ end
9
+ it 'ready' do
10
+ @preview.should be_ready
11
+ end
12
+ it 'values' do
13
+ table = @preview.preview_table
14
+ table.row_value_arrays.first.should == ['Albert','Pujols',29]
15
+ table.row_value_arrays.should size(3)
16
+ end
17
+ it 'keys' do
18
+ @preview.preview_table.keys.should == ['first','last','age']
19
+ end
20
+ end
21
+
22
+ describe 'xml' do
23
+ before do
24
+ @preview = Preview.new(:filename => spec_file('players.xml'))
25
+ #@preview.table_paths = nil
26
+ end
27
+ it 'should need fields' do
28
+ @preview.addl_required_fields.should == [:root_path,:table_paths]
29
+ end
30
+ it 'can set needed fields' do
31
+ @preview.root_path = 'abc'
32
+ @preview.addl_required_fields.should == [:table_paths]
33
+ end
34
+ describe 'ready' do
35
+ before do
36
+ @preview.root_path = 'top'
37
+ @preview.table_paths = [['players','player']]
38
+ end
39
+ it 'ready' do
40
+ @preview.should be_ready
41
+ end
42
+ it 'values' do
43
+ @preview.table.row_value_arrays.first.should == ['Albert','Pujols',29]
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ end
@@ -0,0 +1,39 @@
1
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
2
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
3
+ require 'import_everything'
4
+ require 'spec'
5
+ require 'spec/autorun'
6
+
7
+ Spec::Runner.configure do |config|
8
+
9
+ end
10
+
11
+ include ImportEverything
12
+
13
+ def mit(name,&b)
14
+ it(name,&b) #if name == 'parses values'
15
+ end
16
+
17
+ Spec::Matchers.define :size do |exp_size|
18
+ match do |arr|
19
+ arr.size == exp_size
20
+ end
21
+ failure_message_for_should do |arr|
22
+ "Array is size #{arr.size}, expected #{exp_size}, array is #{arr.inspect}"
23
+ end
24
+ end
25
+
26
+ def pujols_value_hash
27
+ {'first' => 'Albert', 'last' => 'Pujols', 'age' => 29}
28
+ end
29
+
30
+ def pujols_row_hash
31
+ {:table => 'players', :values => pujols_value_hash}
32
+ end
33
+
34
+ def spec_file(f)
35
+ File.expand_path(File.dirname(__FILE__)) + "/#{f}"
36
+ end
37
+
38
+ def mylog(*args)
39
+ end
@@ -0,0 +1,106 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "ImportEverything" do
4
+ describe SqlInsertParser do
5
+ before do
6
+ @pujols_value_hash = pujols_value_hash
7
+ end
8
+ describe SqlInsertParser::LineParser do
9
+ before do
10
+ @sql = "INSERT into players (first ,LAST,age) VALUES (\"Albert\", 'Pujols',29)"
11
+ end
12
+ fattr(:parser) do
13
+ SqlInsertParser::LineParser.new(:line => @sql)
14
+ end
15
+ SqlInsertParser::LineParser.regex_hash.each do |name,reg|
16
+ mit "can match against #{name} regex" do
17
+ (@sql =~ reg).should be
18
+ end
19
+ end
20
+ mit 'parses table name' do
21
+ parser.table.should == 'players'
22
+ end
23
+ mit 'parses raw columns' do
24
+ parser.parsed_elements[:columns].should == 'first ,LAST,age'
25
+ end
26
+ mit 'parses raw values' do
27
+ parser.parsed_elements[:values].should == "\"Albert\", 'Pujols',29"
28
+ end
29
+ mit 'parses columns' do
30
+ parser.columns.should == %w(first last age)
31
+ end
32
+ mit 'parses values' do
33
+ parser.values.should == ['Albert','Pujols',29]
34
+ end
35
+ mit 'value hash' do
36
+ parser.value_hash.should == @pujols_value_hash
37
+ end
38
+ mit 'quote regex match' do
39
+ ('"abc"' =~ /^['"].*['"]$/).should be
40
+ '"abc"'.should =~ /^['"].*['"]$/
41
+ "'abc'".should =~ /^['"].*['"]$/
42
+ end
43
+ it "ignore commented lines" do
44
+ line = "# INSERT INTO players (a,b,c) VALUES (d,e,f)"
45
+ SqlInsertParser::LineParser.should_not be_probable_insert(line)
46
+ end
47
+ it 'can handle quoted table' do
48
+ @sql = "INSERT into 'players' (first ,LAST,'age') VALUES (\"Albert\", 'Pujols',29)"
49
+ parser.table.should == 'players'
50
+ end
51
+ it 'can handle quoted column' do
52
+ @sql = "INSERT into players (first ,LAST,'age') VALUES (\"Albert\", 'Pujols',29)"
53
+ parser.columns.should == %w(first last age)
54
+ end
55
+ it 'can handle multi line insert' do
56
+ @sql = "INSERT into players (first,last,age)
57
+ VALUES (\"Albert\", 'Pujols',29)"
58
+ parser.values.should == ['Albert','Pujols',29]
59
+ parser.value_hash.should == @pujols_value_hash
60
+ end
61
+ end
62
+
63
+ describe 'file parsing' do
64
+ before do
65
+ @lines = ["select * from foo","CREATE TABLE PLAYERS (","insert stuff",")"]
66
+ @lines << "#INSERT into players (first,last,age) VALUES (\"Hanley\", 'Ramirez',27); \n "
67
+ @lines << "INSERT into players (first,last,age) VALUES (\"Albert\", 'Pujols',29)\n"
68
+ @lines << "INSERT into players (first,last,age) VALUES (\"Hanley\", 'Ramirez',27); \n "
69
+ @lines << "INSERT into players (first,last,age) VALUES (\"David\", 'Wright',26); "
70
+ end
71
+ fattr(:raw_sql) { @lines.join("\n") }
72
+ fattr(:sql) do
73
+ File.create(spec_file("players.sql"),raw_sql)
74
+ raw_sql
75
+ end
76
+ fattr(:parser) { SqlInsertParser.new(:str => sql) }
77
+ it 'has right number of lines' do
78
+ parser.lines.size.should == @lines.size
79
+ end
80
+ it 'has 3 probable lines' do
81
+ parser.probable_insert_lines.size.should == 3
82
+ end
83
+ it 'value hashes' do
84
+ #raise parser.insert_lines.inspect
85
+ parser.value_hashes.size.should == 3
86
+ parser.value_hashes.first.should == @pujols_value_hash
87
+ parser.value_hashes[1].should == {'first' => 'Hanley', 'last' => 'Ramirez', 'age' => 27}
88
+ end
89
+ it 'row hashes' do
90
+ parser.row_hashes.first.should == pujols_row_hash
91
+ end
92
+ describe 'multi line insert' do
93
+ before do
94
+ @lines[-3] = @lines[-3].gsub(/VALUES/,"\nVALUES").gsub(/players /,"players\n")
95
+ end
96
+ it 'right number of insert lines' do
97
+ parser.insert_lines.should size(3)
98
+ end
99
+ it 'right value hash' do
100
+ parser.value_hashes.first.should == @pujols_value_hash
101
+ end
102
+ end
103
+ end
104
+ end
105
+
106
+ end
@@ -0,0 +1,56 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ class TestDB
4
+ class << self
5
+ fattr(:instance) { new }
6
+ end
7
+ fattr(:filename) { "/Code/wheeeee.sqlite3" }
8
+ fattr(:db) do
9
+ gem 'sqlite3-ruby'
10
+ require 'sqlite3'
11
+ SQLite3::Database.new(filename)
12
+ #SQLite3::Database.new(open("http://localhost:3000/wheeeee.sqlite3") { |f| f.read })
13
+ end
14
+ fattr(:crefate) do
15
+ `rm -f #{filename}`
16
+ db.execute("CREATE TABLE cities ( name varchar(255) )")
17
+ db.execute("CREATE TABLE players ( first varchar(255), last varchar(255), age integer)")
18
+ #raise db.execute("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;").inspect
19
+ db.execute("INSERT into players (first,last,age) VALUES ('Albert','Pujols',29)")
20
+ db.execute("INSERT into players (first,last,age) VALUES ('David','Wright',26)")
21
+ db.execute("INSERT into players (first,last,age) VALUES ('Hanley','Ramirez',27)")
22
+ db.execute("INSERT into cities (name) VALUES ('Madison')")
23
+ raise "foo" unless db.execute("select count(*) from players").flatten.first.to_i == 3
24
+ end
25
+ def create; end
26
+ end
27
+ def create_test_db!
28
+
29
+ end
30
+
31
+ # raise TestDB.instance.db.execute2("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;").flatten.inspect
32
+
33
+ describe "ImportEverything" do
34
+ describe SqliteParser do
35
+ before do
36
+ TestDB.instance.create
37
+ #@parser = SqliteParser.new(:filename => TestDB.instance.filename)
38
+ @parser = ImportEverything.get_parser(:filename => TestDB.instance.filename)
39
+ @rows = ImportEverything.get_rows(:filename => TestDB.instance.filename)
40
+ end
41
+ it 'tables' do
42
+ @parser.tables.should == ['cities','players']
43
+ end
44
+ it 'value hashes' do
45
+ @parser.cleaned_value_hashes.size.should == 4
46
+ @parser.cleaned_value_hashes[1].should == pujols_value_hash
47
+ end
48
+ it 'row hashes' do
49
+ @parser.cleaned_row_hashes.first.should == {:table => 'cities', :values => {'name' => 'Madison'}}
50
+ @parser.cleaned_row_hashes[1].should == pujols_row_hash
51
+ @rows.first.should == {:table => 'cities', :values => {'name' => 'Madison'}}
52
+ @rows[1].should == pujols_row_hash
53
+ end
54
+ end
55
+
56
+ end
@@ -0,0 +1,31 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
2
+
3
+ describe "ImportEverything" do
4
+ describe XmlParser do
5
+ before do
6
+ @parser = XmlParser.new(:filename => "/Code/import_everything/spec/players.xml", :root_path => 'top', :table_paths => [['players','player'],['cities','city']])
7
+ #@parser.table_paths = nil
8
+ end
9
+ it 'smoke' do
10
+ 2.should == 2
11
+ end
12
+ it 'parsers' do
13
+ @parser.parsers
14
+ end
15
+ it 'row count' do
16
+ @parser.value_hashes.should size(4)
17
+ @parser.cleaned_row_hashes.should size(4)
18
+ end
19
+ it 'first doc' do
20
+ @parser.parsers.first.cleaned_value_hashes.first.should == pujols_value_hash
21
+ end
22
+ it 'hashes' do
23
+ @parser.parsers.first.cleaned_value_hashes
24
+ end
25
+ it 'city' do
26
+ @parser.cleaned_row_hashes.first.should == pujols_row_hash
27
+ @parser.cleaned_row_hashes.last.should == {:table => 'cities', :values => {'name' => 'Madison'}}
28
+ end
29
+ end
30
+
31
+ end
metadata ADDED
@@ -0,0 +1,111 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: import_everything
3
+ version: !ruby/object:Gem::Version
4
+ prerelease: false
5
+ segments:
6
+ - 0
7
+ - 1
8
+ - 0
9
+ version: 0.1.0
10
+ platform: ruby
11
+ authors:
12
+ - Mike Harris
13
+ autorequire:
14
+ bindir: bin
15
+ cert_chain: []
16
+
17
+ date: 2010-05-18 00:00:00 -04:00
18
+ default_executable:
19
+ dependencies:
20
+ - !ruby/object:Gem::Dependency
21
+ name: rspec
22
+ prerelease: false
23
+ requirement: &id001 !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - ">="
26
+ - !ruby/object:Gem::Version
27
+ segments:
28
+ - 0
29
+ version: "0"
30
+ type: :development
31
+ version_requirements: *id001
32
+ description: import everything
33
+ email: mharris717@gmail.com
34
+ executables: []
35
+
36
+ extensions: []
37
+
38
+ extra_rdoc_files:
39
+ - LICENSE
40
+ - README.rdoc
41
+ files:
42
+ - .document
43
+ - .gitignore
44
+ - LICENSE
45
+ - README.rdoc
46
+ - Rakefile
47
+ - VERSION
48
+ - lib/import_everything.rb
49
+ - lib/import_everything/ext.rb
50
+ - lib/import_everything/parsers/csv_parser.rb
51
+ - lib/import_everything/parsers/determine_type.rb
52
+ - lib/import_everything/parsers/line_parser.rb
53
+ - lib/import_everything/parsers/module_methods.rb
54
+ - lib/import_everything/parsers/parser.rb
55
+ - lib/import_everything/parsers/preview.rb
56
+ - lib/import_everything/parsers/sql_parser.rb
57
+ - lib/import_everything/parsers/sqlite_parser.rb
58
+ - lib/import_everything/parsers/table_parser.rb
59
+ - lib/import_everything/parsers/xml_parser.rb
60
+ - spec/bets.html
61
+ - spec/csv_parser_spec.rb
62
+ - spec/howard.html
63
+ - spec/import_everything_spec.rb
64
+ - spec/junk.rb
65
+ - spec/players.csv
66
+ - spec/players.sql
67
+ - spec/players.xml
68
+ - spec/preview_spec.rb
69
+ - spec/spec_helper.rb
70
+ - spec/sql_parser_spec.rb
71
+ - spec/sqlite_parser_spec.rb
72
+ - spec/xml_parser_spec.rb
73
+ has_rdoc: true
74
+ homepage: http://github.com/mharris717/import_everything
75
+ licenses: []
76
+
77
+ post_install_message:
78
+ rdoc_options:
79
+ - --charset=UTF-8
80
+ require_paths:
81
+ - lib
82
+ required_ruby_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ segments:
87
+ - 0
88
+ version: "0"
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ requirements:
91
+ - - ">="
92
+ - !ruby/object:Gem::Version
93
+ segments:
94
+ - 0
95
+ version: "0"
96
+ requirements: []
97
+
98
+ rubyforge_project:
99
+ rubygems_version: 1.3.6
100
+ signing_key:
101
+ specification_version: 3
102
+ summary: import everything
103
+ test_files:
104
+ - spec/csv_parser_spec.rb
105
+ - spec/import_everything_spec.rb
106
+ - spec/junk.rb
107
+ - spec/preview_spec.rb
108
+ - spec/spec_helper.rb
109
+ - spec/sql_parser_spec.rb
110
+ - spec/sqlite_parser_spec.rb
111
+ - spec/xml_parser_spec.rb