activerecord-jdbc-import 0.0.2 → 0.0.3

data/README.md

@@ -9,6 +9,9 @@ and we duck type, so we just want to say 'bulk load this chunk of data'.
 
 This library aims to make loading data fast and easy.  Just call a method, and it loads your data.
 
+Note: The code here works, but it is pretty ugly.  I am still working
+through the implementation.  This is a playground right now.
+
 ## License
 
 MIT.  Do what you want.  If you make changes please contribute them back.
@@ -46,12 +49,48 @@ Or install it yourself as:
     Product.import(products) 
     
     Product.count.should eq(100)
+
+Or, you might want to use plain hashes.  Creating an ActiveRecord object
+for each row might be too much overhead for you:
+
+    require 'active_record/jdbc/import'
+    
+    class Product < ActiveRecord::Base
+        include ActiveRecord::Jdbc::Import
+    end
+    
+    products = []
     
+    1.upto(100) do
+      product = {
+        :name => "foobar123"
+      }
+      products << product
+    end
+    
+    Product.import(products) 
+    
+    Product.count.should eq(100)
+
 It is easy to use, and probably could be easier still.  Feel free to fork the code,
 make changes, fix bugs, etc.
 
+## Tricks
+
+If you are using MySQL, you can speed up the import by adding the
+following options to your database.yml file:
+
+    development:
+      adapter: mysql
+      ...
+      options:
+        useServerPrepStmts: 'false'
+        rewriteBatchedStatements: 'true'
+
 ## TODO
 
+* Support more column types.  Right now, this gem expects text and
+  numeric fields.
 * The `id` column is currently ignored.  The library assumes that `id` is going to be autoincremented. Make this optional. 
 * Test with more databases.  Right now, Teradata, MySQL, and SQLite3 are all working.
 

data/lib/active_record/jdbc/import.rb

@@ -10,41 +10,103 @@ module ActiveRecord
 
       end
 
+      # Code here is ugly.
+      # Going to clean up as soon as the whole idea is worked out.
+
       module ClassMethods
+
+        def import_infile(models)
+          return unless models.size > 0
+
+          start = Time.now
+
+          CSV.open("/tmp/upload.csv", "w") do |row|
+
+            row << models.first.keys.map { |a| a.to_s }
+
+            models.each do |model|
+              row << model.values
+            end
+          end
+
+          sql = []
+          sql << "LOAD DATA INFILE '/tmp/upload.csv'"
+          sql << "INTO TABLE #{self.table_name}"
+          sql << "FIELDS TERMINATED BY ','"
+          sql << "OPTIONALLY ENCLOSED BY '\"'"
+          sql << "LINES TERMINATED BY '\\n'"
+          sql << "IGNORE 1 LINES"
+          sql << "(#{models.first.keys.map { |a| a.to_s }.join(",")});"
+
+          self.connection.execute(sql.join(" "))
+
+          p "Wrote tempfile: #{Time.now - start}"
+
+        end
+
         def import(models)
           return unless models.size > 0
 
           conn = self.connection.jdbc_connection
 
-          insert_sql = models.first.to_prepared_sql
+          o = self.new
+          insert_sql = o.to_prepared_sql
+          ordered_columns = o.ordered_columns
 
           pstmt = conn.prepareStatement(insert_sql)
 
           conn.setAutoCommit(false)
 
-          models.each do |model|
-            i = 1
-            model.attributes.each_pair do |key, value|
-              
-              next if key.to_s == 'id'
-
-              model_type = self.columns_hash[key].type
-              if model_type == :integer and value.nil?
-                pstmt.setInt(i, nil)
-              elsif model_type == :integer
-                pstmt.setInt(i, value.to_i)
-              elsif key.to_s.downcase == 'week_date'
-                pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
-              elsif model_type == :string and value.nil?
-                pstmt.setString(i, nil)
-              else
-                pstmt.setString(i, value.to_s)
+          if models.first.is_a? Hash
+            models.each do |model|
+              i = 1
+              ordered_columns.each do |key|
+                next if key.to_s == 'id'
+                next if key.to_s == 'created_at'
+                next if key.to_s == 'updated_at'
+
+                column_type = self.columns_hash[key.to_s].type
+                value = model[key.to_sym]
+
+                if column_type == :integer and value.nil?
+                  pstmt.setInt(i, nil)
+                elsif column_type == :integer
+                  pstmt.setInt(i, value.to_i)
+                elsif key.to_s.downcase == 'week_date'
+                  pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
+                elsif column_type == :string and value.nil?
+                  pstmt.setString(i, nil)
+                else
+                  pstmt.setString(i, value.to_s)
+                end
+                i += 1
               end
-
-              i += 1
+              pstmt.addBatch()
+            end
+          else
+            models.each do |model|
+              i = 1
+              model.attributes.each_pair do |key, value|
+                next if key.to_s == 'id'
+                next if key.to_s == 'created_at'
+                next if key.to_s == 'updated_at'
+
+                column_type = self.columns_hash[key.to_s].type
+                if column_type == :integer and value.nil?
+                  pstmt.setInt(i, nil)
+                elsif column_type == :integer
+                  pstmt.setInt(i, value.to_i)
+                elsif key.to_s.downcase == 'week_date'
+                  pstmt.setString(i, value.to_date.strftime("%Y/%m/%d"))
+                elsif column_type == :string and value.nil?
+                  pstmt.setString(i, nil)
+                else
+                  pstmt.setString(i, value.to_s)
+                end
+                i += 1
+              end
+              pstmt.addBatch()
             end
-
-            pstmt.addBatch()
           end
 
           pstmt.executeBatch()
@@ -55,23 +117,35 @@ module ActiveRecord
         end
       end
 
+      def ordered_columns
+        @ordered_columns
+      end
+
       def to_prepared_sql
         conn = self.connection
-        
+
+        at_date = DateTime.now
+
         quoted_columns = []
         quoted_values = []
+        @ordered_columns = []
         attributes_with_values = self.send(:arel_attributes_values, true, true)
         attributes_with_values.each_pair do |key,value|
           next if key.name.to_s == 'id'
           quoted_columns << conn.quote_column_name(key.name)
-          quoted_values << '?'
+          if key.name.to_s == 'created_at' or key.name.to_s == 'updated_at'
+            quoted_values << "'#{at_date.to_s(:db)}'"
+          else
+            @ordered_columns << key.name.to_s
+            quoted_values << '?'
+          end
         end
-        
+
         "INSERT INTO #{self.class.quoted_table_name} " +
           "(#{quoted_columns.join(', ')}) "  +
-          "VALUES (#{quoted_values.join(', ')});"
+          "VALUES (#{quoted_values.join(', ')})"
       end
-      
+
     end
   end
 end

data/lib/active_record/jdbc/import/version.rb

@@ -1,7 +1,7 @@
 module Activerecord
   module Jdbc
     module Import
-      VERSION = "0.0.2"
+      VERSION = "0.0.3"
     end
   end
 end

data/spec/import_spec.rb

@@ -27,6 +27,7 @@ describe ActiveRecord::Jdbc::Import do
   end
 
   it 'should import 100 rows of data' do
+    Product.delete_all
     products = []
     1.upto(100) do
       product = Product.new
@@ -42,6 +43,41 @@ describe ActiveRecord::Jdbc::Import do
 
   end
 
+  it 'should import hash data' do
+    Product.delete_all
+    products = []
+    1.upto(100) do
+      product = {
+        :code => Faker::Lorem.word,
+        :name => Faker::Name.name,
+        :vendor => Faker::Name.name,
+        :price => "#{rand(1000)}.#{rand(99)}".to_f
+      }
+      products << product
+    end
+
+    Product.import(products)
+    Product.count.should eq(100)
+  end
+
+  it 'should import hash data - out of order' do
+    Product.delete_all
+    products = []
+    1.upto(100) do
+      product = {
+        :vendor => Faker::Name.name,
+        :name => Faker::Name.name,
+        :price => "#{rand(1000)}.#{rand(99)}".to_f,
+        :code => Faker::Lorem.word
+      }
+      products << product
+    end
+
+    Product.import(products)
+    Product.count.should eq(100)
+  end
+
+
   after(:all) do
     CreateProducts.down
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activerecord-jdbc-import
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease:
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-05-20 00:00:00.000000000 Z
+date: 2013-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport