bulk_update 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bulk_update.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Philip Kurmann
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,72 @@
+# BulkUpdate
+
+Updates a large amount of Records in a highliy efficient way.
+Enhances Active Record with a method for bulk inserts and a method for bulk updates. Both merthods are used for inserting or updating large amount of Records.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'bulk_update'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bulk_update
+
+## Usage
+
+### Bulk insert
+
+Bulk insert inserts a large amount of data into as SQL-Bulk-Inserts. Example:
+    columns = [:name, :value]
+    values  = [['name1', 'value1'], ['name2', 'value2'], ['name3', 'value3']]
+    Model.bulk_insert columns, values
+
+### Bulk update
+
+Bulk update updates a large amount of data. Update means, it creates new records, updates existing records which have changed and deletes old records. This is all done with ActiveRecord which means, all callbacks are executed.
+You have to provide a columns as a key, which is used determine which records are new, have changed or has to be deleted. Only the values provided in the array 'values' are compared and will be updated. Example:
+
+    columns = [:name, :value]
+    values  = [['name1', 'value1'], ['name2', 'value2'], ['name3', 'value3']]
+    Model.bulk_insert columns, values
+
+You have now the following entries in your database:
+<pre>
+  +----+----------------+
+  | id | name  | value  |
+  +----+----------------+
+  |  0 | name1 | value1 |
+  |  1 | name2 | value2 |
+  |  2 | name3 | value3 |
+  +----+----------------+
+</pre>
+
+If you now do a bulk update:
+
+    values  = [['name1', 'value1.1'], ['name2', 'value2'], ['name4', 'value4.1']]
+    Model.bulk_update columns, values, key: 'name'
+
+You have now the following entries in your database:
+<pre>
+  +----+------------------+
+  | id | name  | value    |
+  +----+------------------+
+  |  0 | name1 | value1.1 |
+  |  1 | name2 | value2   |
+  |  3 | name4 | value4.1 |
+  +----+------------------+
+</pre>
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,4 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')

data/bulk_update.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/bulk_update/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Philip Kurmann"]
+  gem.email         = ["philip@kman.ch"]
+  gem.description   = %q{Updates a large amount of Records in a highly efficient way}
+  gem.summary       = %q{Enhances Active Record with a method for bulk inserts and a method for bulk updates. Both merthods are used for inserting or updating large amount of Records.}
+  gem.homepage      = ""
+
+  gem.add_dependency "activerecord"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "sqlite3"
+  gem.add_development_dependency "pry"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "bulk_update"
+  gem.require_paths = ["lib"]
+  gem.version       = BulkUpdate::VERSION
+end

data/lib/bulk_update.rb

@@ -0,0 +1,5 @@
+require "bulk_update/version"
+require "bulk_update/active_record_inflections"
+require "active_record"
+
+ActiveRecord::Base.send(:extend, BulkUpdate::ActiveRecordInflections)

data/lib/bulk_update/active_record_inflections.rb

@@ -0,0 +1,261 @@
+module BulkUpdate
+  module ActiveRecordInflections
+    #
+    # Clone the database structure of a table
+    def clone_table args = {}
+      if args[:to]
+        case ActiveRecord::Base.connection_config[:adapter]
+        when 'sqlite3'
+          ActiveRecord::Base.connection.execute "CREATE TABLE `#{args[:to]}` AS SELECT * FROM `#{table_name}` LIMIT 1"
+          ActiveRecord::Base.connection.execute "DELETE FROM `#{args[:to]}`"
+        else
+          ActiveRecord::Base.connection.execute "CREATE TABLE `#{args[:to]}` LIKE `#{table_name}`"
+        end
+      end
+    end
+
+
+    def insert_str element
+      if element.class == Fixnum || element.class == Float
+        element
+      elsif element.class == NilClass
+        'NULL'
+      else
+        if element.to_s[0] == '(' || element.to_s.downcase == 'true' || element.to_s.downcase == 'false'
+          element.to_s
+        else
+          "'#{element}'"
+        end
+      end
+    end
+
+
+    #
+    # Bulk insert records
+    def bulk_insert columns, values, args = {}
+      # Limit inserts
+      max_records_per_insert = args[:max_records_per_insert] || 100
+      table = args[:into] || table_name
+      columns = columns.clone
+
+      # Add timestamp
+      timestamp = Time.now
+      add_timestamp = false
+      add_updated_at = false
+      unless columns.map(&:to_sym).include?(:created_at)
+        columns << :created_at
+        add_created_at = true
+      end
+      unless columns.map(&:to_sym).include?(:updated_at)
+        columns << :updated_at
+        add_updated_at = true
+      end
+
+      if table_name
+        # Create header for insert with all column names
+        columns = columns.clone.map!{ |c| "`#{c}`" }
+        insert_head = "INSERT INTO `#{table}` (#{columns.join(', ')})"
+
+        # Create inserts
+        inserts = []
+        values.each do |values_per_record|
+          values_per_record = values_per_record.clone
+          values_per_record << timestamp if add_created_at
+          values_per_record << timestamp if add_updated_at
+          inserts << "(#{values_per_record.map{ |e| insert_str(e) }.join(', ')})"
+          if inserts.count > max_records_per_insert
+            ActiveRecord::Base.connection.execute "#{insert_head} VALUES #{inserts.join(', ')}"
+            inserts.clear
+          end
+        end
+        ActiveRecord::Base.connection.execute "#{insert_head} VALUES #{inserts.join(', ')}" unless inserts.empty?
+      end
+    end
+
+
+    #
+    # Create, update and delete Records according to a set of new values through ActiveRecord but optimized for performance by
+    # finding all diferences by SQL.
+    def bulk_update columns, values, args = {}
+      temp_table = "#{table_name}_temp_table_#{$$}"
+      key = args[:key] || args[:keys] || 'id'
+      condition = args[:condition]
+      exclude_fields = args[:exclude_fields]
+      insert = args[:insert].nil? ? true : args[:insert]
+      update = args[:update].nil? ? true : args[:update]
+      remove = args[:remove].nil? ? true : args[:remove]
+
+      # Clone temp-table and load it
+      clone_table to: temp_table
+      bulk_insert columns, values, into: temp_table
+
+      # Find differences and create, update and delete these through ActiveRecord to handle Callbacks, etc.
+      create(get_new_records for: self, compare_with: temp_table, on: key, condition: condition, exclude_fields: exclude_fields) if insert
+      if update
+        get_updated_records(for: self, compare_with: temp_table, on: key, condition: condition, exclude_virtual: args[:exclude_virtual], exclude_fields: exclude_fields).each do |id, new_attributes|
+          find(id).update_attributes new_attributes
+        end
+      end
+      destroy(get_deleted_records for: self, compare_with: temp_table, on: key, condition: condition, exclude_virtual: args[:exclude_virtual]) if remove
+
+    ensure
+      # Drop temp table
+      ActiveRecord::Base.connection.execute "DROP TABLE IF EXISTS #{temp_table}"
+    end
+
+
+    #
+    # Exclude List
+    def default_exclude
+      ['id', 'version', 'created_at', 'updated_at']
+    end
+
+
+    #
+    # Compare Table of args[:model] with its temporary table args[:compare_with] and return all new records as a Array of Hashes
+    def get_new_records args = {}
+      model = args[:for] || self
+      compare_table = args[:compare_with]
+      keys = args[:on] || 'id'
+      exclude = args[:exclude_fields] || []
+      exclude |= default_exclude
+
+      # Generate conditions for query and sub-query
+      conditions = []
+      conditions2 = []
+      conditions << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
+      conditions2 << "#{args[:condition].gsub('--tt--', model.table_name)}" if args[:condition]
+      if keys.class == String || keys.class == Symbol
+        key = keys.to_s
+      else
+        key = keys[0].to_s
+        conditions2 << keys[1..-1].map{|k| "#{model.table_name}.#{k.to_s} = #{compare_table}.#{k.to_s}" }
+      end
+
+      # Generate and execute SQL-Statement
+      condition = conditions.join(' AND ')
+      condition2 = conditions2.join(' AND ')
+      sql = "SELECT * FROM #{compare_table} WHERE #{condition} #{'AND' unless conditions.blank?} #{compare_table}.#{key} NOT IN " +
+            "(SELECT #{key} FROM #{model.table_name} #{'WHERE' unless conditions2.blank?} #{condition2})"
+      results = ActiveRecord::Base.connection.execute sql
+
+      # Generate Array with Hashes of all new records
+      new_records = []
+      keys_to_log = []
+      results.each do |attributes|
+        new_records << result2hash(attributes, exclude)
+        keys_to_log << new_records.last[key.to_sym]
+      end
+      args[:logger].info "New Records for Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
+      new_records
+    end
+
+
+    #
+    # Compare Table of args[:model] with its temporary table args[:compare_with] and return all updated records as a Hash of Hashes whose
+    # key is the ID of the changed record
+    def get_updated_records args = {}
+      model = args[:for] || self
+      compare_table = args[:compare_with]
+      keys = args[:on] || 'id'
+      exclude = args[:exclude_fields] || []
+      exclude |= default_exclude
+      exclude_virtual = args[:exclude_virtual].nil? ? false : args[:exclude_virtual]
+
+      # Generate conditions for query and sub-query
+      conditions = []
+      conditions2 = []
+      conditions << "NOT #{model.table_name}.virtual" if exclude_virtual
+      if keys.class == String || keys.class == Symbol
+        key = keys.to_s
+        conditions << "#{model.table_name}.#{key} = #{compare_table}.#{key}"
+        exclude |= [keys.to_s]
+      else
+        key = keys[0].to_s
+        conditions |= keys.map{|k| "#{model.table_name}.#{k.to_s} = #{compare_table}.#{k.to_s}" }
+        exclude |= keys.map(&:to_s)
+      end
+      conditions << "#{args[:condition].gsub('--tt--', model.table_name)} AND #{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
+      model.attribute_names.each do |an|
+        unless exclude.include?(an)
+          if ActiveRecord::Base.connection_config[:adapter] =~ /mysql?/
+            conditions2 << "NOT #{model.table_name}.#{an} <=> #{compare_table}.#{an}" unless exclude.include?(an)
+          else
+            conditions2 << "NOT (#{model.table_name}.#{an} = #{compare_table}.#{an} OR (#{model.table_name}.#{an} IS NULL AND #{compare_table}.#{an} IS NULL))"
+          end
+        end
+      end
+
+      # Generate and execute SQL-Statement
+      condition = "#{conditions.join(' AND ')} AND (#{conditions2.join(' OR ')})"
+      sql = "SELECT #{model.table_name}.id, #{compare_table}.* FROM #{model.table_name}, #{compare_table} WHERE #{condition}"
+      results = ActiveRecord::Base.connection.execute sql
+
+      # Generate Hash with id as the key and values as a Hashes of all changed records
+      results_hash = {}
+      keys_to_log = []
+      results.each do |attributes|
+        id = attributes[0]
+        results_hash[id] = result2hash attributes, exclude, 1
+        keys_to_log << (args[:debug] ? model.find(id).send(key) : id)
+      end
+      args[:logger].info "Change Records for Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
+
+      results_hash
+    end
+
+
+    #
+    # Compare Table of args[:model] with its temporary table args[:compare_with] and return all deleted records as a Array of IDs
+    def get_deleted_records args = {}
+      model = args[:for] || self
+      compare_table = args[:compare_with]
+      keys = args[:on] || 'id'
+      exclude_virtual = args[:exclude_virtual].nil? ? false : args[:exclude_virtual]
+
+      # Generate conditions for query and sub-query
+      conditions = []
+      conditions2 = []
+      conditions << "NOT #{model.table_name}.virtual" if exclude_virtual
+      conditions << "#{args[:condition].gsub('--tt--', model.table_name)}" if args[:condition]
+      conditions2 << "#{args[:condition].gsub('--tt--', compare_table)}" if args[:condition]
+      if keys.class == String || keys.class == Symbol
+        key = keys.to_s
+      else
+        key = keys[0].to_s
+        conditions2 |= keys[1..-1].map{|k| "#{model.table_name}.#{k.to_s} = #{compare_table}.#{k.to_s}" }
+      end
+
+      # Generate and execute SQL-Statement
+      condition = conditions.join(' AND ') unless conditions.blank?
+      condition2 = conditions2.join(' AND ') unless conditions2.blank?
+      sql = "SELECT id, #{key} FROM #{model.table_name} WHERE #{condition} #{'AND' unless conditions.blank?} #{model.table_name}.#{key} NOT IN " +
+            "(SELECT #{key} FROM #{compare_table} #{'WHERE' unless conditions2.blank?} #{condition2})"
+      results = ActiveRecord::Base.connection.execute sql
+
+      # Generate Array with ids of all deleted records
+      deleted_records = []
+      keys_to_log = []
+      results.each do |attributes|
+        deleted_records << attributes[0]
+        keys_to_log << attributes[1]
+      end
+      args[:logger].info "Deleting Records from Model #{model.to_s}: #{keys_to_log.join(', ')}" unless keys_to_log.blank? || args[:logger].blank?
+      deleted_records
+    end
+
+
+  private
+
+
+    def result2hash attributes, exclude, attribute_nr = 0
+      hash = {}
+      attribute_names.each do |an|
+        hash[an.to_sym] = attributes[attribute_nr] unless exclude.include?(an)
+        attribute_nr += 1
+      end
+      hash
+    end
+
+  end
+end

data/lib/bulk_update/version.rb

@@ -0,0 +1,3 @@
+module BulkUpdate
+  VERSION = "1.0.0"
+end

data/spec/bulk_update_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper.rb'
+
+require 'pry'
+
+describe BulkUpdate do
+  before :each do
+    @columns = [:name, :value]
+    @values  = [['test1', 'value1'], ['test2', 'value2'], ['test3', 'value3'], ['test4', 'value4']]
+    MyHash.bulk_insert @columns, @values
+  end
+
+
+  it 'inserts multiple records in one SQL' do
+    MyHash.count.should be 4
+  end
+
+
+  it 'updates and deletes records' do
+    @values  = [['test1', 'value1.1'], ['test2', 'value2'], ['test4', 'value4.4'], ['test5', 'value5.5']]
+    MyHash.bulk_update @columns, @values, key: 'name'
+    MyHash.count.should be 4
+    MyHash.where(name: 'test1').first.value.should eq 'value1.1'
+    MyHash.where(name: 'test2').first.value.should eq 'value2'
+    MyHash.where(name: 'test3').first.should be nil
+    MyHash.where(name: 'test4').first.value.should eq 'value4.4'
+    MyHash.where(name: 'test5').first.value.should eq 'value5.5'
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'support/active_record'
+require 'support/my_hash'
+
+require 'bulk_update'
+
+
+RSpec.configure do |config|
+  config.color_enabled = true
+  config.formatter     = 'documentation'
+end

data/spec/support/active_record.rb

@@ -0,0 +1,39 @@
+require 'active_record'
+require 'active_record/version'
+puts "Testing ActiveRecord #{ActiveRecord::VERSION::STRING}"
+
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+ActiveRecord::Migrator.up "db/migrate"
+
+load 'spec/support/schema.rb'
+
+
+module ActiveModel::Validations
+  # Extension to enhance `should have` on AR Model instances.  Calls
+  # model.valid? in order to prepare the object's errors object.
+  #
+  # You can also use this to specify the content of the error messages.
+  #
+  # @example
+  #
+  #     model.should have(:no).errors_on(:attribute)
+  #     model.should have(1).error_on(:attribute)
+  #     model.should have(n).errors_on(:attribute)
+  #
+  #     model.errors_on(:attribute).should include("can't be blank")
+  def errors_on(attribute)
+    self.valid?
+    [self.errors[attribute]].flatten.compact
+  end
+  alias :error_on :errors_on
+end
+
+
+RSpec.configure do |config|
+  config.around do |example|
+    ActiveRecord::Base.transaction do
+      example.run
+      raise ActiveRecord::Rollback
+    end
+  end
+end

data/spec/support/my_hash.rb

@@ -0,0 +1,4 @@
+require 'active_record'
+
+class MyHash < ActiveRecord::Base
+end

data/spec/support/schema.rb

@@ -0,0 +1,5 @@
+ActiveRecord::Migration.create_table :my_hashes do |t|
+  t.string :name
+  t.string :value
+  t.timestamps
+end

metadata

@@ -0,0 +1,129 @@
+--- !ruby/object:Gem::Specification
+name: bulk_update
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Philip Kurmann
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Updates a large amount of Records in a highly efficient way
+email:
+- philip@kman.ch
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bulk_update.gemspec
+- lib/bulk_update.rb
+- lib/bulk_update/active_record_inflections.rb
+- lib/bulk_update/version.rb
+- spec/bulk_update_spec.rb
+- spec/spec_helper.rb
+- spec/support/active_record.rb
+- spec/support/my_hash.rb
+- spec/support/schema.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Enhances Active Record with a method for bulk inserts and a method for bulk
+  updates. Both merthods are used for inserting or updating large amount of Records.
+test_files:
+- spec/bulk_update_spec.rb
+- spec/spec_helper.rb
+- spec/support/active_record.rb
+- spec/support/my_hash.rb
+- spec/support/schema.rb