load_data_infile 0.1.0

data/.document

@@ -0,0 +1,5 @@
+README.rdoc
+lib/**/*.rb
+bin/*
+features/**/*.feature
+LICENSE

data/.gitignore

@@ -0,0 +1,21 @@
+## MAC OS
+.DS_Store
+
+## TEXTMATE
+*.tmproj
+tmtags
+
+## EMACS
+*~
+\#*
+.\#*
+
+## VIM
+*.swp
+
+## PROJECT::GENERAL
+coverage
+rdoc
+pkg
+
+## PROJECT::SPECIFIC

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Emmanuel Oga
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,26 @@
+= load_data_infile
+
+This is a plugin for ActiveRecord.
+
+Some chunks of code for this library were taken from activerecord-fast-import
+(http://github.com/jsuchal/activerecord-fast-import)
+
+It provides a mean of calling MySql's LOAD DATA INFILE clause for importing a csv file without having to proces it with ruby.
+
+For details see MySql's manual
+
+http://dev.mysql.com/doc/refman/5.1/en/load-data.html
+
+== Note on Patches/Pull Requests
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2010 Emmanuel Oga. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,45 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "load_data_infile"
+    gem.summary = %Q{MySQL LOAD DATA INFILE support for ActiveRecord}
+    gem.description = %Q{MySQL LOAD DATA INFILE support for ActiveRecord}
+    gem.email = "EmmanuelOga@gmail.com"
+    gem.homepage = "http://github.com/EmmanuelOga/load_data_infile"
+    gem.authors = ["Emmanuel Oga"]
+    gem.add_development_dependency "rspec", ">= 1.2.9"
+    # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
+end
+
+require 'spec/rake/spectask'
+Spec::Rake::SpecTask.new(:spec) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.spec_files = FileList['spec/**/*_spec.rb']
+end
+
+Spec::Rake::SpecTask.new(:rcov) do |spec|
+  spec.libs << 'lib' << 'spec'
+  spec.pattern = 'spec/**/*_spec.rb'
+  spec.rcov = true
+end
+
+task :spec => :check_dependencies
+
+task :default => :spec
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "load_data_infile #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/lib/load_data_infile.rb

@@ -0,0 +1,113 @@
+require 'erb'
+
+module LoadDataInfile
+  module MySql
+
+    # Deletes all rows in table very fast, but without calling +destroy+ method
+    # nor any hooks.
+    def truncate_table(table = quoted_table_name)
+      connection.execute("TRUNCATE TABLE #{table}")
+    end
+
+    # Disables key updates for model table
+    def disable_keys(table = quoted_table_name)
+      connection.execute("ALTER TABLE #{table} DISABLE KEYS")
+    end
+
+    # Enables key updates for model table
+    def enable_keys(table = quoted_table_name)
+      connection.execute("ALTER TABLE #{table} ENABLE KEYS")
+    end
+
+    # Disables keys, yields block, enables keys.
+    def with_keys_disabled(table = quoted_table_name)
+      disable_keys(table)
+      yield
+    ensure
+      enable_keys(table)
+    end
+
+    # Load csv from a file using MySql's LOAD DATA INFILE
+    # For details see: http://dev.mysql.com/doc/refman/5.1/en/load-data.html
+    #
+    # Options:
+    #
+    # path                   :: CSV file path
+    #
+    # charset                :: [OPTIONAL] Charset
+    # columns                :: [OPTIONAL] Array of columns. Tries to use all columns if not provided. Use @dummy as column name to ignore a column. E.G.: (column_a, @column_b, @dummy)
+    # concurrent             :: [OPTIONAL] True or false
+    # enclosed_by            :: [OPTIONAL] Character
+    # escaped_by             :: [OPTIONAL] Character
+    # ignore                 :: [OPTIONAL] Number, If provided, skips that number of lines.
+    # lines_starting_by      :: [OPTIONAL] Character
+    # lines_terminated_by    :: [OPTIONAL] Character
+    # local                  :: [OPTIONAL] true or fase. Defaults to true
+    # low_priority           :: [OPTIONAL] true or false
+    # mappings               :: [OPTIONAL] An array to map column values according to the mysql manual. E.G.: { :column_a => "TRIM(@column_b)"}
+    # on_duplicates          :: [OPTIONAL] Action to perform when a duplicate row is found. Can be IGNORE or REPLACE
+    # optionally_enclosed_by :: [OPTIONAL] Character
+    # table                  :: [OPTIONAL] Table name. Defaults to quoted_table_name (won't work if used from an abstract class, e.g. ActiveRecord::Base')
+    # terminated_by          :: [OPTIONAL] Character
+    def load_data_infile(options = {})
+      c = Context.new
+
+      if options[:low_priority]
+        c.low_priority_or_concurrent = :LOW_PRIORITY
+      elsif options[:concurrent]
+        c.low_priority_or_concurrent = :CONCURRENT
+      end
+
+      c.local = :LOCAL if !options.member?(:local) || options[:local]
+
+      c.file_name = quote_value options[:path]
+
+      c.replace_or_ignore = options[:on_duplicates] if options[:on_duplicates] # REPLACE or IGNORE
+
+      c.table_name = options[:table] ? "`#{ options[:table] }`" : quoted_table_name
+
+      c.charset = "CHARACTER SET #{quote_value options[:charset]}" if options[:charset]
+
+      if options[:terminated_by] || options[:enclosed_by] || options[:optionally_enclosed_by] || options[:escaped_by]
+        c.fields_definitions = " FIELDS " # or COLUMNS
+        c.fields_definitions << " TERMINATED          BY #{ quote_value options[:terminated_by] } "          if options[:terminated_by]
+        c.fields_definitions << " ENCLOSED            BY #{ quote_value options[:enclosed_by] } "            if options[:enclosed_by]
+        c.fields_definitions << " OPTIONALLY ENCLOSED BY #{ quote_value options[:optionally_enclosed_by] } " if options[:optionally_enclosed_by]
+        c.fields_definitions << " ESCAPED             BY #{ quote_value options[:escaped_by] } "             if options[:escaped_by]
+      end
+
+      if options[:lines_terminated_by] || options[:lines_starting_by]
+        c.lines_defitions = " LINES "
+        c.lines_defitions << " STARTING BY   #{quote_value options[:lines_starting_by]} "   if options[:lines_starting_by]
+        c.lines_defitions << " TERMINATED BY #{quote_value options[:lines_terminated_by]} " if options[:lines_terminated_by]
+      end
+
+      c.ignores = "IGNORE #{options[:ignore]} LINES" if options[:ignore]
+
+      c.columns = " (#{options[:columns].join(", ")}) " if options[:columns]
+
+      if options[:mappings] && options[:mappings].length > 0
+        s = options[:mappings].map{|column, mapping| "#{column} = #{mapping}" }.join(",")
+        c.mappings = "SET #{s}"
+      end
+
+      connection.execute(ERB.new(LOAD_DATA_INFILE_SQL).result(c.binding).gsub(/^\s*\n/, ""))
+    end
+
+    class Context < OpenStruct
+      public :binding
+    end
+
+    LOAD_DATA_INFILE_SQL = <<-SQL
+      LOAD DATA <%= low_priority_or_concurrent %> <%= local %> INFILE <%= file_name %>
+          <%= replace_or_ignore %>
+          INTO TABLE <%= table_name %>
+          <%= charset %>
+          <%= fields_definitions %>
+          <%= lines_defitions %>
+          <%= ignores %>
+          <%= columns %>
+          <%= mappings %> ;
+    SQL
+  end
+end

data/rails/init.rb

@@ -0,0 +1 @@
+ActiveRecord::Base.extend LoadDataInfile::MySql

data/spec/active_record_helper.rb

@@ -0,0 +1,13 @@
+ActiveRecord::Base.establish_connection(:adapter => "mysql", :database => "load_data_infile_test", :user => "root", :password => "")
+
+ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+ActiveRecord::Schema.define do
+  create_table "things", :force => true do |t|
+    t.string :field_a, :field_b
+    t.integer :field_c
+  end
+end
+
+class Thing < ActiveRecord::Base
+end

data/spec/fixtures/csv_with_headers.csv

@@ -0,0 +1,2 @@
+id,a,b,c
+71,Hello,Brother,42

data/spec/fixtures/csv_without_headers.csv

@@ -0,0 +1 @@
+61,live,from,2400

data/spec/load_data_infile_spec.rb

@@ -0,0 +1,61 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe LoadDataInfile do
+  before :each do
+    Thing.truncate_table
+  end
+
+  it "loads data from a csv file with headers into an ActiveRecord table" do
+    Thing.with_keys_disabled do
+      Thing.load_data_infile(
+        :path          => FIXTURE_WITH_HEADERS,
+        :columns       => %w|id field_a field_b field_c|,
+        :terminated_by => ",",
+        :ignore        => 1
+      )
+    end
+    Thing.all.map(&:attributes).should == [{
+      "id"      => 71,
+      "field_a" => "Hello",
+      "field_b" => "Brother",
+      "field_c" => 42
+    }]
+  end
+
+  it "loads data from a csv file without headers into an ActiveRecord table" do
+    Thing.with_keys_disabled do
+      Thing.load_data_infile(
+        :path          => FIXTURE_WITHOUT_HEADERS,
+        :terminated_by => ",",
+        :columns       => %w|id field_a field_b field_c|
+      )
+    end
+    Thing.all.map(&:attributes).should == [{
+      "id"      => 61,
+      "field_a" => "live",
+      "field_b" => "from",
+      "field_c" => 2400
+    }]
+  end
+
+  it "loads data from a csv file with mapping" do
+    Thing.with_keys_disabled do
+      Thing.load_data_infile(
+        :path          => FIXTURE_WITHOUT_HEADERS,
+        :terminated_by => ",",
+        :columns       => %w|id @field_a @field_b @field_c|,
+        :mappings      => {
+                            :field_a => "CONCAT('So ', @field_a)",
+                            :field_b => "CONCAT('Much ', @field_b)",
+                            :field_c => "@field_c * 10",
+                          }
+      )
+    end
+    Thing.all.map(&:attributes).should == [{
+      "id"      => 61,
+      "field_a" => "So live",
+      "field_b" => "Much from",
+      "field_c" => 24000
+    }]
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,16 @@
+SPEC_PATH = File.dirname(__FILE__)
+$LOAD_PATH.unshift(SPEC_PATH)
+$LOAD_PATH.unshift(File.join(SPEC_PATH, '..', 'lib'))
+require 'load_data_infile'
+require 'spec'
+require 'spec/autorun'
+require 'rubygems'
+require 'active_record'
+require 'active_record_helper'
+require File.join(SPEC_PATH, "..", "rails", "init.rb")
+
+FIXTURE_WITH_HEADERS = File.join(SPEC_PATH, "fixtures", "csv_with_headers.csv")
+FIXTURE_WITHOUT_HEADERS = File.join(SPEC_PATH, "fixtures", "csv_without_headers.csv")
+
+Spec::Runner.configure do |config|
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: load_data_infile
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Emmanuel Oga
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-02-23 00:00:00 -03:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 2
+        - 9
+        version: 1.2.9
+  type: :development
+  version_requirements: *id001
+description: MySQL LOAD DATA INFILE support for ActiveRecord
+email: EmmanuelOga@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- lib/load_data_infile.rb
+- rails/init.rb
+- spec/active_record_helper.rb
+- spec/fixtures/csv_with_headers.csv
+- spec/fixtures/csv_without_headers.csv
+- spec/load_data_infile_spec.rb
+- spec/spec_helper.rb
+has_rdoc: true
+homepage: http://github.com/EmmanuelOga/load_data_infile
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: MySQL LOAD DATA INFILE support for ActiveRecord
+test_files: 
+- spec/load_data_infile_spec.rb
+- spec/spec_helper.rb
+- spec/active_record_helper.rb