bulk_data_methods 1.0.0

data/.gitignore

@@ -0,0 +1,24 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile ~/.gitignore_global
+
+# Ignore bundler config
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+
+# Ignore all logfiles and tempfiles.
+/log/*.log
+/tmp
+.idea/*
+database.yml
+/log/*.pid
+spec/dummy/db/*.sqlite3
+spec/dummy/log/*.log
+spec/dummy/tmp/
+spec/dummy/.sass-cache
+spec/dummy/config/database.yml
+spec/dummy/db/schema.rb

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in bulk_data_methods.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,108 @@
+PATH
+  remote: .
+  specs:
+    bulk_data_methods (1.0.0)
+      pg
+      rails (>= 3.0.0)
+      rspec-rails
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actionmailer (3.2.8)
+      actionpack (= 3.2.8)
+      mail (~> 2.4.4)
+    actionpack (3.2.8)
+      activemodel (= 3.2.8)
+      activesupport (= 3.2.8)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      journey (~> 1.0.4)
+      rack (~> 1.4.0)
+      rack-cache (~> 1.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.1.3)
+    activemodel (3.2.8)
+      activesupport (= 3.2.8)
+      builder (~> 3.0.0)
+    activerecord (3.2.8)
+      activemodel (= 3.2.8)
+      activesupport (= 3.2.8)
+      arel (~> 3.0.2)
+      tzinfo (~> 0.3.29)
+    activeresource (3.2.8)
+      activemodel (= 3.2.8)
+      activesupport (= 3.2.8)
+    activesupport (3.2.8)
+      i18n (~> 0.6)
+      multi_json (~> 1.0)
+    arel (3.0.2)
+    builder (3.0.3)
+    diff-lcs (1.1.3)
+    erubis (2.7.0)
+    hike (1.2.1)
+    i18n (0.6.1)
+    journey (1.0.4)
+    json (1.7.5)
+    mail (2.4.4)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    mime-types (1.19)
+    multi_json (1.3.6)
+    pg (0.14.1)
+    polyglot (0.3.3)
+    rack (1.4.1)
+    rack-cache (1.2)
+      rack (>= 0.4)
+    rack-ssl (1.3.2)
+      rack
+    rack-test (0.6.1)
+      rack (>= 1.0)
+    rails (3.2.8)
+      actionmailer (= 3.2.8)
+      actionpack (= 3.2.8)
+      activerecord (= 3.2.8)
+      activeresource (= 3.2.8)
+      activesupport (= 3.2.8)
+      bundler (~> 1.0)
+      railties (= 3.2.8)
+    railties (3.2.8)
+      actionpack (= 3.2.8)
+      activesupport (= 3.2.8)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (>= 0.14.6, < 2.0)
+    rake (0.9.2.2)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.11.0)
+      rspec-core (~> 2.11.0)
+      rspec-expectations (~> 2.11.0)
+      rspec-mocks (~> 2.11.0)
+    rspec-core (2.11.1)
+    rspec-expectations (2.11.3)
+      diff-lcs (~> 1.1.3)
+    rspec-mocks (2.11.3)
+    rspec-rails (2.11.0)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec (~> 2.11.0)
+    sprockets (2.1.3)
+      hike (~> 1.2)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    thor (0.16.0)
+    tilt (1.3.3)
+    treetop (1.4.10)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.33)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bulk_data_methods!

data/LICENSE

@@ -0,0 +1,30 @@
+Copyright (c) 2010-2012, Fiksu, Inc.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+ o Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+
+ o Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the
+   distribution.
+
+ o Fiksu, Inc. nor the names of its contributors may be used to
+   endorse or promote products derived from this software without
+   specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README

@@ -0,0 +1,68 @@
+bulk_data_methods
+==================
+MixIn used to extend ActiveRecord::Base classes implementing bulk insert and update operations
+through {#create_many} and {#update_many}.
+
+Examples
+========
+
+class Company < ActiveRecord::Base
+  extend BulkMethodsMixin
+end
+__________________________
+BULK creation of many rows:
+
+example no options used
+  rows = [
+             { :name => 'Keith', :salary => 1000 },
+             { :name => 'Alex', :salary => 2000 }
+         ]
+  Employee.create_many(rows)
+
+example with :returning option to returns key value
+  rows = [
+             { :name => 'Keith', :salary => 1000 },
+             { :name => 'Alex', :salary => 2000 }
+         ]
+  options = { :returning => [:id] }
+  Employee.create_many(rows, options)
+
+example with :slice_size option (will generate two insert queries)
+  rows = [
+             { :name => 'Keith', :salary => 1000 },
+             { :name => 'Alex', :salary => 2000 },
+             { :name => 'Mark', :salary => 3000 }
+       ]
+  options = { :slice_size => 2 }
+  Employee.create_many(rows, options)
+_________________________
+BULK updates of many rows:
+
+example using "set_array" to add the value of "salary" to the specific employee's salary the default where clause matches IDs so, it works here.
+  rows = [
+             { :id => 1, :salary => 1000 },
+             { :id => 10, :salary => 2000 },
+             { :id => 23, :salary => 2500 }
+       ]
+  options = { :set_array => '"salary = datatable.salary"' }
+  Employee.update_many(rows, options)
+
+example using where clause to match salary.
+  rows = [
+             { :id => 1, :salary => 1000, :company_id => 10 },
+             { :id => 10, :salary => 2000, :company_id => 12 },
+             { :id => 23, :salary => 2500, :company_id => 5 }
+       ]
+  options = {
+              :set_array => '"company_id = datatable.company_id"',
+              :where => '"#{table_name}.salary = datatable.salary"'
+            }
+  Employee.update_many(rows, options)
+
+  example setting where clause to the KEY of the hash passed in and the set_array is generated from the VALUES
+  rows = {
+             { :id => 1 } => { :salary => 100000, :company_id => 10 },
+             { :id => 10 } => { :salary => 110000, :company_id => 12 },
+             { :id => 23 } => { :salary => 90000, :company_id => 5 }
+       }
+  Employee.update_many(rows)

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bulk_data_methods.gemspec

@@ -0,0 +1,20 @@
+$LOAD_PATH.push File.expand_path("../lib", __FILE__)
+require 'bulk_data_methods/version'
+
+Gem::Specification.new do |s|
+  s.name         = "bulk_data_methods"
+  s.version      = BulkDataMethods::VERSION
+  s.license      = 'New BSD License'
+  s.date         = '2012-09-21'
+  s.summary      = 'MixIn used to extend ActiveRecord::Base classes implementing bulk insert and update operations through {#create_many} and {#update_many}.'
+  s.description  = 'MixIn used to extend ActiveRecord::Base classes implementing bulk insert and update operations through {#create_many} and {#update_many}.'
+  s.authors      = ["Keith Gabryelski"]
+  s.email        = 'keith@fiksu.com'
+  s.files        = `git ls-files`.split("\n")
+  s.test_files   = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.require_path = 'lib'
+  s.homepage     = 'http://github.com/fiksu/bulk_data_methods'
+  s.add_dependency "pg"
+  s.add_dependency "rails", '>= 3.0.0'
+  s.add_dependency 'rspec-rails'
+end

data/lib/bulk_data_methods/bulk_methods_mixin.rb

@@ -0,0 +1,228 @@
+# MixIn used to extend ActiveRecord::Base classes implementing bulk insert and update operations
+# through {#create_many} and {#update_many}.
+# @example to use:
+#   class Company < ActiveRecord::Base
+#     extend BulkMethodsMixin
+#   end
+#
+module BulkMethodsMixin
+  # exception thrown when row data structures are inconsistent between rows in single call to {#create_many} or {#update_many}
+  class BulkUploadDataInconsistent < StandardError
+    def initialize(model, table_name, expected_columns, found_columns, while_doing)
+      super("#{model.name}: for table: #{table_name}; #{expected_columns} != #{found_columns}; #{while_doing}")
+    end
+  end
+
+  # BULK creation of many rows
+  #
+  # @example no options used
+  #   rows = [
+  #         { :name => 'Keith', :salary => 1000 },
+  #         { :name => 'Alex', :salary => 2000 }
+  #   ]
+  #   Employee.create_many(rows)
+  #
+  # @example with :returning option to returns key value
+  #   rows = [
+  #         { :name => 'Keith', :salary => 1000 },
+  #         { :name => 'Alex', :salary => 2000 }
+  #   ]
+  #   options = { :returning => [:id] }
+  #   Employee.create_many(rows, options)
+  #   [#<Employee id: 1>, #<Employee id: 2>]
+  #
+  # @example with :slice_size option (will generate two insert queries)
+  #   rows = [
+  #         { :name => 'Keith', :salary => 1000 },
+  #         { :name => 'Alex', :salary => 2000 },
+  #         { :name => 'Mark', :salary => 3000 }
+  #   ]
+  #   options = { :slice_size => 2 }
+  #   Employee.create_many(rows, options)
+  #
+  # @param [Array<Hash>] rows ([]) data to be inserted into database
+  # @param [Hash] options ({}) options for bulk inserts
+  # @option options [Integer] :slice_size (1000) how many records will be created in a single SQL query
+  # @option options [Boolean] :check_consitency (true) ensure some modicum of sanity on the incoming dataset, specifically: does each row define the same set of key/value pairs
+  # @option options [Array or String] :returning (nil) list of fields to return.
+  # @return [Array<Hash>] rows returned from DB as option[:returning] requests
+  # @raise [BulkUploadDataInconsistent] raised when key/value pairs between rows are inconsistent (check disabled with option :check_consistency)
+  def create_many(rows, options = {})
+    return [] if rows.blank?
+    options[:slice_size] = 1000 unless options.has_key?(:slice_size)
+    options[:check_consistency] = true unless options.has_key?(:check_consistency)
+    returning_clause = ""
+    if options[:returning]
+      if options[:returning].is_a? Array
+        returning_list = options[:returning].join(',')
+      else
+        returning_list = options[:returning]
+      end
+      returning_clause = " returning #{returning_list}"
+    end
+    returning = []
+
+    created_at_value = Time.zone.now
+
+    num_sequences_needed = rows.reject{|r| r[:id].present?}.length
+    if num_sequences_needed > 0
+      row_ids = connection.next_sequence_values(sequence_name, num_sequences_needed)
+    else
+      row_ids = []
+    end
+    rows.each do |row|
+      # set the primary key if it needs to be set
+      row[:id] ||= row_ids.shift
+    end.each do |row|
+      # set :created_at if need be
+      row[:created_at] ||= created_at_value
+    end.group_by do |row|
+      respond_to?(:partition_table_name) ? partition_table_name(*partition_key_values(row)) : table_name
+    end.each do |table_name, rows_for_table|
+      column_names = rows_for_table[0].keys.sort{|a,b| a.to_s <=> b.to_s}
+      sql_insert_string = "insert into #{table_name} (#{column_names.join(',')}) values "
+      rows_for_table.map do |row|
+        if options[:check_consistency]
+          row_column_names = row.keys.sort{|a,b| a.to_s <=> b.to_s}
+          if column_names != row_column_names
+            raise BulkUploadDataInconsistent.new(self, table_name, column_names, row_column_names, "while attempting to build insert statement")
+          end
+        end
+        column_values = column_names.map do |column_name|
+          quote_value(row[column_name], columns_hash[column_name.to_s])
+        end.join(',')
+        "(#{column_values})"
+      end.each_slice(options[:slice_size]) do |insert_slice|
+        returning += find_by_sql(sql_insert_string + insert_slice.join(',') + returning_clause)
+      end
+    end
+    return returning
+  end
+
+  #
+  # BULK updates of many rows
+  #
+  # @return [Array<Hash>] rows returned from DB as option[:returning] requests
+  # @raise [BulkUploadDataInconsistent] raised when key/value pairs between rows are inconsistent (check disabled with option :check_consistency)
+  # @param [Hash] options ({}) options for bulk inserts
+  # @option options [Integer] :slice_size (1000) how many records will be created in a single SQL query
+  # @option options [Boolean] :check_consitency (true) ensure some modicum of sanity on the incoming dataset, specifically: does each row define the same set of key/value pairs
+  # @option options [Array] :returning (nil) list of fields to return.
+  # @option options [String] :returning (nil) single field to return.
+  #
+  # @overload update_many(rows = [], options = {})
+  #   @param [Array<Hash>] rows ([]) data to be updated
+  #   @option options [String] :set_array (built from first row passed in) the set clause
+  #   @option options [String] :where ('"#{table_name}.id = datatable.id"') the where clause
+  #
+  # @overload update_many(rows = {}, options = {})
+  #   @param [Hash<Hash, Hash>] rows ({}) data to be updated
+  #   @option options [String] :set_array (built from the values in the first key/value pair of `rows`) the set clause
+  #   @option options [String] :where (built from the keys in the first key/value pair of `rows`) the where clause
+  #
+  # @example using "set_array" to add the value of "salary" to the specific employee's salary the default where clause matches IDs so, it works here.
+  #   rows = [
+  #     { :id => 1, :salary => 1000 },
+  #     { :id => 10, :salary => 2000 },
+  #     { :id => 23, :salary => 2500 }
+  #   ]
+  #   options = { :set_array => '"salary = datatable.salary"' }
+  #   Employee.update_many(rows, options)
+  #
+  # @example using where clause to match salary.
+  #   rows = [
+  #     { :id => 1, :salary => 1000, :company_id => 10 },
+  #     { :id => 10, :salary => 2000, :company_id => 12 },
+  #     { :id => 23, :salary => 2500, :company_id => 5 }
+  #   ]
+  #   options = {
+  #     :set_array => '"company_id = datatable.company_id"',
+  #     :where => '"#{table_name}.salary = datatable.salary"'
+  #   }
+  #   Employee.update_many(rows, options)
+  #
+  # @example setting where clause to the KEY of the hash passed in and the set_array is generated from the VALUES
+  #   rows = {
+  #     { :id => 1 } => { :salary => 100000, :company_id => 10 },
+  #     { :id => 10 } => { :salary => 110000, :company_id => 12 },
+  #     { :id => 23 } => { :salary => 90000, :company_id => 5 }
+  #   }
+  #   Employee.update_many(rows)
+  #
+  # @note Remember that you should probably set updated_at using "updated = datatable.updated_at"
+  #   or "updated_at = now()" in the set_array if you want to follow
+  #   the standard active record model for time columns (and you have an updated_at column)
+  def update_many(rows, options = {})
+    return [] if rows.blank?
+    if rows.is_a?(Hash)
+      options[:where] = '"' + rows.keys[0].keys.map{|key| '#{table_name}.' + "#{key} = datatable.#{key}"}.join(' and ') + '"'
+      options[:set_array] = '"' + rows.values[0].keys.map{|key| "#{key} = datatable.#{key}"}.join(',') + '"' unless options[:set_array]
+      r = []
+      rows.each do |key,value|
+        r << key.merge(value)
+      end
+      rows = r
+    end
+    unless options[:set_array]
+      column_names =  rows[0].keys
+      columns_to_remove = [:id]
+      columns_to_remove += [partition_keys].map{|k| k.to_sym} if respond_to?(:partition_keys)
+      options[:set_array] = '"' + (column_names - columns_to_remove.flatten).map{|cn| "#{cn} = datatable.#{cn}"}.join(',') + '"'
+    end
+    options[:slice_size] = 1000 unless options[:slice_size]
+    options[:check_consistency] = true unless options.has_key?(:check_consistency)
+    returning_clause = ""
+    if options[:returning]
+      if options[:returning].is_a?(Array)
+        returning_list = options[:returning].map{|r| '#{table_name}.' + r.to_s}.join(',')
+      else
+        returning_list = options[:returning]
+      end
+      returning_clause = "\" returning #{returning_list}\""
+    end
+    options[:where] = '"#{table_name}.id = datatable.id"' unless options[:where]
+
+    returning = []
+
+    rows.group_by do |row|
+      respond_to?(:partition_table_name) ? partition_table_name(*partition_key_values(row)) : table_name
+    end.each do |table_name, rows_for_table|
+      column_names = rows_for_table[0].keys.sort{|a,b| a.to_s <=> b.to_s}
+      rows_for_table.each_slice(options[:slice_size]) do |update_slice|
+        datatable_rows = []
+        update_slice.each_with_index do |row,i|
+          if options[:check_consistency]
+            row_column_names = row.keys.sort{|a,b| a.to_s <=> b.to_s}
+            if column_names != row_column_names
+              raise BulkUploadDataInconsistent.new(self, table_name, column_names, row_column_names, "while attempting to build update statement")
+            end
+          end
+          datatable_rows << row.map do |column_name,column_value|
+            column_name = column_name.to_s
+            columns_hash_value = columns_hash[column_name]
+            if i == 0
+              "#{quote_value(column_value, columns_hash_value)}::#{columns_hash_value.sql_type} as #{column_name}"
+            else
+              quote_value(column_value, columns_hash_value)
+            end
+          end.join(',')
+        end
+        datatable = datatable_rows.join(' union select ')
+
+        sql_update_string = <<-SQL
+          update #{table_name} set
+            #{eval(options[:set_array])}
+          from
+          (select
+            #{datatable}
+          ) as datatable
+          where
+            #{eval(options[:where])}
+          #{eval(returning_clause)}
+        SQL
+        returning += find_by_sql(sql_update_string)
+      end
+    end
+    return returning
+  end
+end

data/lib/bulk_data_methods/monkey_patch_postgres.rb

@@ -0,0 +1,30 @@
+require 'active_record'
+require 'active_record/base'
+require 'active_record/connection_adapters/abstract_adapter'
+require 'active_record/connection_adapters/postgresql_adapter'
+
+module ActiveRecord::ConnectionAdapters
+  class PostgreSQLAdapter < AbstractAdapter
+    #
+    # Get the next value in a sequence. Used on INSERT operation for
+    # partitioning like by_id because the ID is required before the insert
+    # so that the specific child table is known ahead of time.
+    #
+    # @param [String] sequence_name the name of the sequence to fetch the next value from
+    # @return [Integer] the value from the sequence
+    def next_sequence_value(sequence_name)
+      return execute("select nextval('#{sequence_name}')").field_values("nextval").first.to_i
+    end
+
+    #
+    # Get the some next values in a sequence.
+    #
+    # @param [String] sequence_name the name of the sequence to fetch the next values from
+    # @param [Integer] batch_size count of values.
+    # @return [Array<Integer>] an array of values from the sequence
+    def next_sequence_values(sequence_name, batch_size)
+      result = execute("select nextval('#{sequence_name}') from generate_series(1, #{batch_size})")
+      return result.field_values("nextval").map(&:to_i)
+    end
+  end
+end

data/lib/bulk_data_methods/version.rb

@@ -0,0 +1,3 @@
+module BulkDataMethods
+  VERSION = "1.0.0"
+end

data/lib/bulk_data_methods.rb

@@ -0,0 +1,3 @@
+require "bulk_data_methods/version"
+require "bulk_data_methods/bulk_methods_mixin"
+require "bulk_data_methods/monkey_patch_postgres"