guillaumegentil-import_fu 0.1.0

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Thibaud Guillaume-Gentil
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,14 @@
+require 'rubygems'
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'echoe'
+
+Echoe.new('import_fu', '0.1.0') do |p|
+  p.description    = "Add quick mass data import from CSV or Array to Active Record model"
+  p.url            = "http://github.com/guillaumegentil/import_fu/tree"
+  p.author         = "Thibaud Guillaume-Gentil"
+  p.email          = "guillaumegentil@gmail.com"
+  p.ignore_pattern = ["tmp/*", "script/*"]
+  p.development_dependencies = ["faster_csv", "active_record"]
+end

data/import_fu.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{import_fu}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Thibaud Guillaume-Gentil"]
+  s.date = %q{2008-12-11}
+  s.description = %q{Add quick mass data import from CSV or Array to Active Record model}
+  s.email = %q{guillaumegentil@gmail.com}
+  s.extra_rdoc_files = ["lib/import_fu.rb", "README"]
+  s.files = ["init.rb", "lib/import_fu.rb", "Manifest", "MIT-LICENSE", "Rakefile", "README", "test/import_fu_test.rb", "test/test_helper.rb", "import_fu.gemspec"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/guillaumegentil/import_fu/tree}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Import_fu", "--main", "README.markdown"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{import_fu}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{Add quick mass data import from CSV or Array to Active Record model}
+  s.test_files = ["test/import_fu_test.rb", "test/test_helper.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<faster_csv>, [">= 0"])
+      s.add_development_dependency(%q<active_record>, [">= 0"])
+    else
+      s.add_dependency(%q<faster_csv>, [">= 0"])
+      s.add_dependency(%q<active_record>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<faster_csv>, [">= 0"])
+    s.add_dependency(%q<active_record>, [">= 0"])
+  end
+end

data/init.rb

@@ -0,0 +1 @@
+require 'import_fu'

data/lib/import_fu.rb

@@ -0,0 +1,75 @@
+require 'fastercsv'
+
+# Adds a import_with_load_data_infile class method.
+# this lets you import data using mysql "LOAD DATA INFILE"  
+# This is about 30% faster than using ar-extensions bulk import
+# 
+# be careful, there's no validation or escaping here!
+module ImportFu
+  
+  def self.included(base)
+    base.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    
+    def import(*args)
+      options = { :local => false, :replace => false, :timestamps => true, :format => false }
+      options.merge!(args.pop) if args.last.is_a? Hash
+      
+      columns = args.first
+      if args.last.is_a? Array # Array of values
+        columns += [:created_at, :updated_at] if options[:timestamps]
+        csv_tempfile = create_csv_tempfile_for(args.last, options)
+        csv_path = csv_tempfile.path
+      else # path of an existing csv
+        csv_path = args.last
+      end
+      
+      load_data_infile_sql = build_load_data_infile_statement(csv_path, columns, options)
+      ActiveRecord::Base.connection.execute(load_data_infile_sql)
+      
+      csv_tempfile.close! if csv_tempfile
+    end
+    
+    protected
+    
+    def create_csv_tempfile_for(values, options = {})
+      # change permissions on tmpdir
+      File.new(Dir::tmpdir).chmod(0755) rescue nil
+      # create tempfile
+      csv_tempfile = Tempfile.new('ImportFu')
+      csv_tempfile.chmod(0644)
+      # write csv in tempfile
+      FasterCSV.open(csv_tempfile.path, "w") do |csv|
+        values.each do |column_values|
+          column_values.map! { |value| format_value(value) } if options[:format]
+          if options[:timestamps]
+            csv << (column_values + [Time.now.utc.to_s(:db), Time.now.utc.to_s(:db)])
+          else
+            csv << column_values
+          end
+        end
+      end
+      csv_tempfile
+    end 
+    
+    def format_value(value)
+      case value.class.to_s
+      when "Time", "Date"
+        value.to_s(:db)
+      else
+        value.to_s
+      end
+    end
+    
+    def build_load_data_infile_statement(csv_path, columns, options = {})
+      local = options[:local] ? 'LOCAL' : ''
+      replace = options[:replace] ? 'REPLACE' : 'IGNORE'
+      column_list = columns.map(&:to_s).join(',')
+      
+      "LOAD DATA #{local} INFILE '#{csv_path}' #{replace} INTO TABLE #{table_name} FIELDS TERMINATED BY ',' ENCLOSED BY '\"' (#{column_list});"
+    end
+    
+  end
+end

data/test/import_fu_test.rb

@@ -0,0 +1,118 @@
+require File.expand_path(File.dirname(__FILE__) + '/test_helper')
+
+# reopen the module and make methods public for testing
+module ImportFu
+  module ClassMethods
+    public :create_csv_tempfile_for
+    public :build_load_data_infile_statement
+  end 
+end 
+
+class ImportFuTest < Test::Unit::TestCase
+  
+  def setup 
+    rebuild_model
+  end     
+  
+  def test_should_respond_to_import
+    assert Foo.respond_to?(:import)
+  end
+  
+  def test_should_create_a_temp_file_with_the_csv_content
+    # columns = [:name, :size]
+    values = [["toto", "32"],
+              ["tata", "33"]]
+  
+    file = Foo.create_csv_tempfile_for(values)
+    
+    assert_equal IO.read(file.path), IO.read(fixtures('foos.csv'))
+  end
+  
+  def test_should_import_data_form_array
+    columns = [:name, :size]
+    values = [["toto", "32"],
+              ["tata", "33"]]
+    
+    Foo.import columns, values
+    
+    assert_equal 2, Foo.count
+    assert_not_nil Foo.find_by_name_and_size('toto', 32)
+    assert_not_nil Foo.find_by_name_and_size('tata', 33)
+  end
+  
+  def test_should_import_data_form_array_with_timestamps
+    columns = [:name, :size]
+    values = [["toto", "32"]]
+    
+    Foo.import columns, values
+    
+    assert_kind_of Time, Foo.find_by_name_and_size('toto', 32).created_at
+    assert_kind_of Time, Foo.find_by_name_and_size('toto', 32).updated_at
+  end
+  
+  def test_should_import_data_form_array_without_timestamps
+    columns = [:name, :size]
+    values = [["toto", "32"]]
+    
+    Foo.import columns, values, :timestamps => false
+    
+    assert_nil Foo.find_by_name_and_size('toto', 32).created_at
+    assert_nil Foo.find_by_name_and_size('toto', 32).updated_at
+  end
+  
+  def test_should_import_data_form_array_and_format_value
+    columns = [:name, :size, :created_at, :updated_at]
+    values = [["toto", "32", Time.now.utc, Time.now.utc]]
+    
+    Foo.import columns, values, :timestamps => false, :format => true
+    
+    assert_kind_of Time, Foo.find_by_name_and_size('toto', 32).created_at
+    assert_kind_of Time, Foo.find_by_name_and_size('toto', 32).updated_at
+  end
+  
+  def test_should_replace_imported_data_form_array
+    Foo.create(:name => 'toto', :size => '32')
+    Foo.create(:name => 'tata', :size => '33')
+    
+    columns = [:id, :name, :size]
+    values = [["1", "toto", "1000"],
+              ["2", "tata", "1001"]]
+    
+    Foo.import columns, values, :replace => true
+    
+    assert_equal 2, Foo.count
+    
+    assert_nil Foo.find_by_name_and_size('toto', 32)
+    assert_nil Foo.find_by_name_and_size('tata', 33)
+    assert_not_nil Foo.find_by_name_and_size('toto', 1000)
+    assert_not_nil Foo.find_by_name_and_size('tata', 1001)
+  end
+
+  def test_should_import_data_form_csv
+    columns = [:name, :size]
+    values = [["toto", "32"],
+              ["tata", "33"]]
+    
+    Foo.import columns, fixtures('foos.csv')
+    
+    assert_equal 2, Foo.count
+    assert_not_nil Foo.find_by_name_and_size('toto', 32)
+    assert_not_nil Foo.find_by_name_and_size('tata', 33)
+  end
+  
+  def test_should_generate_the_correct_default_import_statement
+    expected = "LOAD DATA  INFILE 'fake.csv' IGNORE INTO TABLE foos FIELDS TERMINATED BY ',' ENCLOSED BY '\"' (col1,col2);"
+    assert_equal expected, Foo.build_load_data_infile_statement('fake.csv', [:col1,:col2]) 
+  end
+  
+  def test_should_generate_the_correct_import_statement_with_locale
+    expected = "LOAD DATA LOCAL INFILE 'fake.csv' IGNORE INTO TABLE foos FIELDS TERMINATED BY ',' ENCLOSED BY '\"' (col1,col2);"
+    assert_equal expected, Foo.build_load_data_infile_statement('fake.csv', [:col1,:col2], :local => true) 
+  end
+  
+  def test_should_generate_the_correct_import_statement_with_replace
+    expected = "LOAD DATA  INFILE 'fake.csv' REPLACE INTO TABLE foos FIELDS TERMINATED BY ',' ENCLOSED BY '\"' (col1,col2);"
+    assert_equal expected, Foo.build_load_data_infile_statement('fake.csv', [:col1,:col2], :replace => true) 
+  end 
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'test/unit'
+require 'active_record'
+require 'import_fu'
+
+# gem install redgreen for colored test output
+begin require 'redgreen'; rescue LoadError; end
+
+config = YAML::load(IO.read(File.dirname(__FILE__) + '/database.yml'))
+ActiveRecord::Base.establish_connection(config['test'])
+
+def rebuild_model
+  ActiveRecord::Base.connection.create_table :foos, :force => true do |t|
+    t.string :name
+    t.integer :size
+    t.timestamps
+  end
+  
+  ActiveRecord::Base.send(:include, ImportFu)
+  Object.send(:remove_const, "Foo") rescue nil
+  Object.const_set("Foo", Class.new(ActiveRecord::Base))
+  Foo.class_eval do
+    include ImportFu
+  end
+end
+
+def fixtures(path)
+  "#{File.expand_path(File.dirname(__FILE__))}/fixtures/#{path}"
+end

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: guillaumegentil-import_fu
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Thibaud Guillaume-Gentil
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-12-11 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: faster_csv
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: active_record
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Add quick mass data import from CSV or Array to Active Record model
+email: guillaumegentil@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- lib/import_fu.rb
+- README
+files: 
+- init.rb
+- lib/import_fu.rb
+- Manifest
+- MIT-LICENSE
+- Rakefile
+- README
+- test/import_fu_test.rb
+- test/test_helper.rb
+- import_fu.gemspec
+has_rdoc: true
+homepage: http://github.com/guillaumegentil/import_fu/tree
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- Import_fu
+- --main
+- README.markdown
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "1.2"
+  version: 
+requirements: []
+
+rubyforge_project: import_fu
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Add quick mass data import from CSV or Array to Active Record model
+test_files: 
+- test/import_fu_test.rb
+- test/test_helper.rb