bulk_insert2 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3e6a484a5e8f122b794605f2664ca20a5d0c887cf78f05581bf40c2aaaa61970
+  data.tar.gz: 4196d55dfd00c9d571bf8608257807a7fe4d290b4dc90b67fb390f66a58a9303
+SHA512:
+  metadata.gz: 457740006d28c583a70f71ba63c4abf81e4bd875f9fec9dc468f35605a558fc4b610d5a09f921f7e57fbbf9759aece0a008dbc6095a2f7e406ae196580517978
+  data.tar.gz: d338d37cabb673e8417273832229e252988648a8e5d653b428bc77526797ce21789fc2673b519506a159f1319249827d6701d15d13448b558b8f432be55d9ac3

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2015 Jamis Buck
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,199 @@
+### Replaced type\_cast\_from\_column to lookup\_cast\_type\_from\_column
+
+# BulkInsert
+
+A little ActiveRecord extension for helping to insert lots of rows in a
+single insert statement.
+
+## Installation
+
+Add it to your Gemfile:
+
+```ruby
+gem 'bulk_insert'
+```
+
+## Usage
+
+BulkInsert adds a new class method to your ActiveRecord models:
+
+```ruby
+class Book < ActiveRecord::Base
+end
+
+book_attrs = ... # some array of hashes, for instance
+Book.bulk_insert do |worker|
+  book_attrs.each do |attrs|
+    worker.add(attrs)
+  end
+end
+```
+
+All of those `#add` calls will be accumulated into a single SQL insert
+statement, vastly improving the performance of multiple sequential
+inserts (think data imports and the like).
+
+If you don't like using a block API, you can also simply pass an array
+of rows to be inserted:
+
+```ruby
+book_attrs = ... # some array of hashes, for instance
+Book.bulk_insert values: book_attrs
+```
+
+By default, the columns to be inserted will be all columns in the table,
+minus the `id` column, but if you want, you can explicitly enumerate
+the columns:
+
+```ruby
+Book.bulk_insert(:title, :author) do |worker|
+  # specify a row as an array of values...
+  worker.add ["Eye of the World", "Robert Jordan"]
+
+  # or as a hash
+  worker.add title: "Lord of Light", author: "Roger Zelazny"
+end
+```
+
+It will automatically set `created_at`/`updated_at` columns to the current
+date, as well.
+
+```ruby
+Book.bulk_insert(:title, :author, :created_at, :updated_at) do |worker|
+  # specify created_at/updated_at explicitly...
+  worker.add ["The Chosen", "Chaim Potok", Time.now, Time.now]
+
+  # or let BulkInsert set them by default...
+  worker.add ["Hello Ruby", "Linda Liukas"]
+end
+```
+
+Similarly, if a value is omitted, BulkInsert will use whatever default
+value is defined for that column in the database:
+
+```ruby
+# create_table :books do |t|
+#   ...
+#   t.string "medium", default: "paper"
+#   ...
+# end
+
+Book.bulk_insert(:title, :author, :medium) do |worker|
+  worker.add title: "Ender's Game", author: "Orson Scott Card"
+end
+
+Book.first.medium #-> "paper"
+```
+
+By default, the batch is always saved when the block finishes, but you
+can explicitly save inside the block whenever you want, by calling
+`#save!` on the worker:
+
+```ruby
+Book.bulk_insert do |worker|
+  worker.add(...)
+  worker.add(...)
+
+  worker.save!
+
+  worker.add(...)
+  #...
+end
+```
+
+That will save the batch as it has been defined to that point, and then
+empty the batch so that you can add more rows to it if you want. Note
+that all records saved together will have the same created_at/updated_at
+timestamp (unless one was explicitly set).
+
+### Batch Set Size
+
+By default, the size of the insert is limited to 500 rows at a time.
+This is called the _set size_. If you add another row that causes the
+set to exceed the set size, the insert statement is automatically built
+and executed, and the batch is reset.
+
+If you want a larger (or smaller) set size, you can specify it in
+two ways:
+
+```ruby
+# specify set_size when initializing the bulk insert...
+Book.bulk_insert(set_size: 100) do |worker|
+  # ...
+end
+
+# specify it on the worker directly...
+Book.bulk_insert do |worker|
+  worker.set_size = 100
+  # ...
+end
+```
+
+### Insert Ignore
+
+By default, when an insert fails the whole batch of inserts fail. The
+_ignore_ option ignores the inserts that would have failed (because of
+duplicate keys or a null in column with a not null constraint) and
+inserts the rest of the batch.
+
+This is not the default because no errors are raised for the bad
+inserts in the batch.
+
+```ruby
+destination_columns = [:title, :author]
+
+# Ignore bad inserts in the batch
+Book.bulk_insert(*destination_columns, ignore: true) do |worker|
+  worker.add(...)
+  worker.add(...)
+  # ...
+end
+```
+
+### Update Duplicates (MySQL, PostgreSQL)
+
+If you don't want to ignore duplicate rows but instead want to update them
+then you can use the _update_duplicates_ option. Set this option to true
+(MySQL) or list unique column names (PostgreSQL) and when a duplicate row
+is found the row will be updated with your new values.
+Default value for this option is false.
+
+```ruby
+destination_columns = [:title, :author]
+
+# Update duplicate rows (MySQL)
+Book.bulk_insert(*destination_columns, update_duplicates: true) do |worker|
+  worker.add(...)
+  worker.add(...)
+  # ...
+end
+
+# Update duplicate rows (PostgreSQL)
+Book.bulk_insert(*destination_columns, update_duplicates: %w[title]) do |worker|
+  worker.add(...)
+  # ...
+end
+```
+
+### Return Primary Keys (PostgreSQL, PostGIS)
+
+If you want the worker to store primary keys of inserted records, then you can
+use the _return_primary_keys_ option. The worker will store a `result_sets`
+array of `ActiveRecord::Result` objects. Each `ActiveRecord::Result` object
+will contain the primary keys of a batch of inserted records.
+
+```ruby
+worker = Book.bulk_insert(*destination_columns, return_primary_keys: true) do
+|worker|
+  worker.add(...)
+  worker.add(...)
+  # ...
+end
+
+worker.result_sets
+```
+
+## License
+
+BulkInsert is released under the MIT license (see MIT-LICENSE) by
+Jamis Buck (jamis@jamisbuck.org).

data/Rakefile

@@ -0,0 +1,29 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'BulkInsert'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/lib/bulk_insert.rb

@@ -0,0 +1,38 @@
+require 'bulk_insert/worker'
+
+module BulkInsert
+  extend ActiveSupport::Concern
+
+  module ClassMethods
+    def bulk_insert(*columns, values: nil, set_size:500, ignore: false, update_duplicates: false, return_primary_keys: false)
+      columns = default_bulk_columns if columns.empty?
+      worker = BulkInsert::Worker.new(connection, table_name, primary_key, columns, set_size, ignore, update_duplicates, return_primary_keys)
+
+      if values.present?
+        transaction do
+          worker.add_all(values)
+          worker.save!
+        end
+        nil
+      elsif block_given?
+        transaction do
+          yield worker
+          worker.save!
+        end
+        nil
+      else
+        worker
+      end
+    end
+
+    # helper method for preparing the columns before a call to :bulk_insert
+    def default_bulk_columns
+      self.column_names - %w(id)
+    end
+
+  end
+end
+
+ActiveSupport.on_load(:active_record) do
+  send(:include, BulkInsert)
+end

data/lib/bulk_insert/statement_adapters.rb

@@ -0,0 +1,22 @@
+require_relative 'statement_adapters/generic_adapter'
+require_relative 'statement_adapters/mysql_adapter'
+require_relative 'statement_adapters/postgresql_adapter'
+require_relative 'statement_adapters/sqlite_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    def adapter_for(connection)
+      case connection.adapter_name
+      when /^mysql/i
+        MySQLAdapter.new
+      when /\APost(?:greSQL|GIS)/i
+        PostgreSQLAdapter.new
+      when /\ASQLite/i
+        SQLiteAdapter.new
+      else
+        GenericAdapter.new
+      end
+    end
+    module_function :adapter_for
+  end
+end

data/lib/bulk_insert/statement_adapters/base_adapter.rb

@@ -0,0 +1,21 @@
+module BulkInsert
+  module StatementAdapters
+    class BaseAdapter
+      def initialize
+        raise "You cannot initialize base adapter" if self.class == BaseAdapter
+      end
+
+      def insert_ignore_statement
+        raise "Not implemented"
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        raise "Not implemented"
+      end
+
+      def primary_key_return_statement(_primary_key)
+        raise "Not implemented"
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/generic_adapter.rb

@@ -0,0 +1,19 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class GenericAdapter < BaseAdapter
+      def insert_ignore_statement
+        ''
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        ''
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/mysql_adapter.rb

@@ -0,0 +1,24 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class MySQLAdapter < BaseAdapter
+      def insert_ignore_statement
+        'IGNORE'
+      end
+
+      def on_conflict_statement(columns, _ignore, update_duplicates)
+        return '' unless update_duplicates
+
+        update_values = columns.map do |column|
+          "`#{column.name}`=VALUES(`#{column.name}`)"
+        end.join(', ')
+        ' ON DUPLICATE KEY UPDATE ' + update_values
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/postgresql_adapter.rb

@@ -0,0 +1,28 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class PostgreSQLAdapter < BaseAdapter
+      def insert_ignore_statement
+        ''
+      end
+
+      def on_conflict_statement(columns, ignore, update_duplicates)
+        if ignore
+          ' ON CONFLICT DO NOTHING'
+        elsif update_duplicates
+          update_values = columns.map do |column|
+            "#{column.name}=EXCLUDED.#{column.name}"
+          end.join(', ')
+          ' ON CONFLICT(' + update_duplicates.join(', ') + ') DO UPDATE SET ' + update_values
+        else
+          ''
+        end
+      end
+
+      def primary_key_return_statement(primary_key)
+        " RETURNING #{primary_key}"
+      end
+    end
+  end
+end

data/lib/bulk_insert/statement_adapters/sqlite_adapter.rb

@@ -0,0 +1,19 @@
+require_relative 'base_adapter'
+
+module BulkInsert
+  module StatementAdapters
+    class SQLiteAdapter < BaseAdapter
+      def insert_ignore_statement
+        'OR IGNORE'
+      end
+
+      def on_conflict_statement(_columns, _ignore, _update_duplicates)
+        ''
+      end
+
+      def primary_key_return_statement(_primary_key)
+        ''
+      end
+    end
+  end
+end

data/lib/bulk_insert/version.rb

@@ -0,0 +1,7 @@
+module BulkInsert
+  MAJOR = 1
+  MINOR = 0
+  TINY  = 0
+
+  VERSION = [MAJOR, MINOR, TINY].join(".")
+end