bulk_update 1.0.0 → 1.1.0

data/README.md

@@ -22,13 +22,15 @@ Or install it yourself as:
 ### Bulk insert
 
 Bulk insert inserts a large amount of data into as SQL-Bulk-Inserts. Example:
+
     columns = [:name, :value]
     values  = [['name1', 'value1'], ['name2', 'value2'], ['name3', 'value3']]
     Model.bulk_insert columns, values
 
 ### Bulk update
 
-Bulk update updates a large amount of data. Update means, it creates new records, updates existing records which have changed and deletes old records. This is all done with ActiveRecord which means, all callbacks are executed.
+Bulk update updates a large amount of data. Update means, it creates new records, updates existing records which have changed and deletes old records. This is all done with ActiveRecord which means, all callbacks are executed. The difference between the old and the new records are fully discovered by SQL.
+
 You have to provide a columns as a key, which is used determine which records are new, have changed or has to be deleted. Only the values provided in the array 'values' are compared and will be updated. Example:
 
     columns = [:name, :value]
@@ -37,13 +39,13 @@ You have to provide a columns as a key, which is used determine which records ar
 
 You have now the following entries in your database:
 <pre>
-  +----+----------------+
-  | id | name  | value  |
-  +----+----------------+
-  |  0 | name1 | value1 |
-  |  1 | name2 | value2 |
-  |  2 | name3 | value3 |
-  +----+----------------+
++----+-------+--------+-----------+------------+
+| id | name  | value  | timestamp | updated_at |
++----+-------+--------+-----------+------------+
+|  0 | name1 | value1 |        t1 |         t1 |
+|  1 | name2 | value2 |        t1 |         t1 |
+|  2 | name3 | value3 |        t1 |         t1 |
++----+-------+--------+-----------+------------+
 </pre>
 
 If you now do a bulk update:
@@ -53,15 +55,28 @@ If you now do a bulk update:
 
 You have now the following entries in your database:
 <pre>
-  +----+------------------+
-  | id | name  | value    |
-  +----+------------------+
-  |  0 | name1 | value1.1 |
-  |  1 | name2 | value2   |
-  |  3 | name4 | value4.1 |
-  +----+------------------+
++----+-------+----------+-----------+------------+
+| id | name  | value    | timestamp | updated_at |
++----+-------+----------+-----------+------------+
+|  0 | name1 | value1.1 |        t1 |         t2 |
+|  1 | name2 | value2   |        t1 |         t1 |
+|  3 | name4 | value4.1 |        t2 |         t2 |
++----+-------+----------+-----------+------------+
 </pre>
 
+## How it works
+
+### Bulk insert
+
+Bulk insert uses the bulk insert feature of SQL (`INSERT INTO tbl_name (col1,col2,col3) VALUES(v1,v2,v3),(v4,v5,v6),(v7,v8,v9);`)
+
+You can specify the max records per insert with the argument `:max_records_per_insert` (default is 100). After these amount of records, the SQL is sent to the database.
+
+### Bulk update
+
+First, a temp-table will be created with the same structure as the original table. Then, all new records are inserted by bulk-inserts into the temp-table.
+When the temp table was loaded, the difference between these 2 tables are discovered in 3 steps: New records, changed records and deleted records.
+All new, changed and deleted records are handled by ActiveRecord. This ensures, all callbacks get executed as defined.
 
 ## Contributing
 

data/lib/bulk_update/active_record_inflections.rb

@@ -8,6 +8,8 @@ module BulkUpdate
         when 'sqlite3'
           ActiveRecord::Base.connection.execute "CREATE TABLE `#{args[:to]}` AS SELECT * FROM `#{table_name}` LIMIT 1"
           ActiveRecord::Base.connection.execute "DELETE FROM `#{args[:to]}`"
+        when 'postgresql'
+          ActiveRecord::Base.connection.execute "CREATE TABLE \"#{args[:to]}\" (LIKE \"#{table_name}\" INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING INDEXES);"
         else
           ActiveRecord::Base.connection.execute "CREATE TABLE `#{args[:to]}` LIKE `#{table_name}`"
         end
@@ -53,8 +55,14 @@ module BulkUpdate
 
       if table_name
         # Create header for insert with all column names
-        columns = columns.clone.map!{ |c| "`#{c}`" }
-        insert_head = "INSERT INTO `#{table}` (#{columns.join(', ')})"
+        case ActiveRecord::Base.connection_config[:adapter]
+        when 'postgresql'
+          columns = columns.map! { |c| "\"#{c}\"" }
+          insert_head = "INSERT INTO \"#{table}\" (#{columns.join(', ')})"
+        else
+          columns = columns.map!{ |c| "`#{c}`" }
+          insert_head = "INSERT INTO `#{table}` (#{columns.join(', ')})"
+        end
 
         # Create inserts
         inserts = []
@@ -143,6 +151,7 @@ module BulkUpdate
       new_records = []
       keys_to_log = []
       results.each do |attributes|
+        attributes = attributes2array(attributes)
         new_records << result2hash(attributes, exclude)
         keys_to_log << new_records.last[key.to_sym]
       end
@@ -188,15 +197,17 @@ module BulkUpdate
 
       # Generate and execute SQL-Statement
       condition = "#{conditions.join(' AND ')} AND (#{conditions2.join(' OR ')})"
-      sql = "SELECT #{model.table_name}.id, #{compare_table}.* FROM #{model.table_name}, #{compare_table} WHERE #{condition}"
+      compare_columns = attribute_names.select { |e| e != 'id' }.map { |e| "#{compare_table}.#{e}" }.join(', ')
+      sql = "SELECT #{model.table_name}.id, #{compare_columns} FROM #{model.table_name}, #{compare_table} WHERE #{condition}"
       results = ActiveRecord::Base.connection.execute sql
 
       # Generate Hash with id as the key and values as a Hashes of all changed records
       results_hash = {}
       keys_to_log = []
       results.each do |attributes|
+        attributes = attributes2array(attributes)
         id = attributes[0]
-        results_hash[id] = result2hash attributes, exclude, 1
+        results_hash[id] = result2hash attributes, exclude
         keys_to_log << (args[:debug] ? model.find(id).send(key) : id)
       end
       args[:logger].info "Change Records for Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
@@ -237,6 +248,7 @@ module BulkUpdate
       deleted_records = []
       keys_to_log = []
       results.each do |attributes|
+        attributes = attributes2array(attributes)
         deleted_records << attributes[0]
         keys_to_log << attributes[1]
       end
@@ -248,7 +260,7 @@ module BulkUpdate
   private
 
 
-    def result2hash attributes, exclude, attribute_nr = 0
+    def result2hash(attributes, exclude, attribute_nr = 0)
       hash = {}
       attribute_names.each do |an|
         hash[an.to_sym] = attributes[attribute_nr] unless exclude.include?(an)
@@ -257,5 +269,10 @@ module BulkUpdate
       hash
     end
 
+
+    def attributes2array(attributes)
+      ActiveRecord::Base.connection_config[:adapter] == 'postgresql' ? attributes.map{ |e| e.last } : attributes
+    end
+
   end
 end

data/lib/bulk_update/version.rb

@@ -1,3 +1,3 @@
 module BulkUpdate
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bulk_update
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-29 00:00:00.000000000 Z
+date: 2013-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord