rimportor 0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 11df56cdb93fc78219c02060fa2ad2dbbf944217
+  data.tar.gz: e5f6b49bc437eff5e28ad65c08da8a98d6314f1f
+SHA512:
+  metadata.gz: fa2bddde54b617c167030ba41bb7a7a79dde80eeccf3455287cf227eda27440a2b4bbe9cea4307f9b0d2ad711d4d3d608c9507faf84b36f0fa9bcd12d62e14e1
+  data.tar.gz: 1e849d21abee6a89233174355441bb78d22cfc2996da0a501fb0cfd893333bba1bc97bc71b9777e1c2a6d1ea6dfa6c187631c64f7077b3a9101ec7f0b33b267e

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2015 Erwin Schens
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,76 @@
+# Rimportor
+
+Rimportor is a new and modern bulk importing gem.
+It utilizes arel under the hood to generate insert statements.
+By working directly on the model Rimportor is able to execute callbacks and validate the records before inserting them into the database - **which is missing in most importing gems**.
+
+### Features
+ - Import in batches
+ - Validation of the bulk
+ - Callback execution
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rimportor', '~> 0.3'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rimportor
+
+## Usage
+Rimportor adds to every ActiveRecord model an additional method called rimport. This method then takes a collection of your records you want to persist.
+Let me give you an example.
+```ruby
+users = []
+1000.times.each { User.new(some_params) }
+User.rimport users # Imports your collection as a bulk insert to your database
+```
+But wait... what about validations and callbacks of my bulk?
+Rimportor got you! Just add some configuration options for your rimport.
+Let me show you what i mean.
+```ruby
+users = []
+1000.times.each { User.new(some_params) }
+
+# true if bulk valid and imported else false
+User.rimport users, before_callbacks: true, 
+                    after_callbacks: true, 
+                    validate_bulk: true 
+```
+The rimport method returns true if your bulk is valid and all callbacks are executed. 
+If an error occurs Rimportor won't insert your bulk in the database. 
+
+And what if i want to insert my records in batches? Rimportor got your back on that too.
+```ruby
+users = []
+1000.times.each { User.new(some_params) }
+    
+# Rimportor will insert the 1000 records in 100 chunks
+User.rimport users, batch_size: 100
+```
+
+## Benchmarks
+
+In progress...
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release` to create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,23 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Rimportor'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+
+
+
+
+
+Bundler::GemHelper.install_tasks
+

data/lib/generators/install_generator.rb

@@ -0,0 +1,11 @@
+require 'rails/generators'
+
+module Rimportor
+  class InstallGenerator < ::Rails::Generators::Base
+    source_root(File.expand_path(File.dirname(__FILE__)))
+
+    def copy_initializer
+      copy_file '../templates/rimportor.rb', 'config/initializers/rimportor.rb'
+    end
+  end
+end

data/lib/rimportor/active_record/adapter/mysql2.rb

@@ -0,0 +1,40 @@
+module Rimportor
+  module ActiveRecord
+    module Adapter
+      class Mysql2
+
+        def max_allowed_packet
+          exec_in_pool do |connection|
+            result = connection.execute("SHOW VARIABLES like 'max_allowed_packet';")
+            val = result.respond_to?(:fetch_row) ? result.fetch_row[1] : result.first[1]
+            val.to_i
+          end
+        end
+
+        def exec_in_pool
+          ::Rimportor::Util::Connection.in_pool do |connection|
+            yield(connection)
+          end
+        end
+
+        def statement_too_big?(statement)
+          statement.size > max_allowed_packet
+        end
+
+        def exec_insert(import_statement)
+          insert_statement, value_statements = import_statement
+          if statement_too_big? ("#{insert_statement}, #{value_statements.join(',')}")
+            puts 'Statement too big'
+          else
+            exec_statement "#{insert_statement},#{value_statements.join(',')}"
+          end
+        end
+
+        def exec_statement(statement)
+          exec_in_pool { |connection| connection.execute statement }
+        end
+
+      end
+    end
+  end
+end

data/lib/rimportor/active_record/import.rb

@@ -0,0 +1,70 @@
+require 'parallel'
+
+module Rimportor
+  module ActiveRecord
+    class Import
+
+      def initialize(bulk, adapter, opts = {})
+        @bulk = bulk
+        @adapter = adapter
+        @before_callbacks = !!opts[:before_callbacks]
+        @after_callbacks = !!opts[:after_callbacks]
+        @validate_bulk = !!opts[:validate_bulk]
+        @batch_size = opts[:batch_size] ? opts[:batch_size] : 1000
+        @threads = Rimportor.configuration.threads
+      end
+
+      def run_before_callbacks
+        ::Parallel.map(@bulk, in_threads: @threads) do |element|
+          execute_callbacks(element, :before)
+        end
+      end
+
+      def run_after_callbacks
+        ::Parallel.map(@bulk, in_threads: @threads) do |element|
+          execute_callbacks(element, :after)
+        end
+      end
+
+      def run_validations
+        validation_result = ::Parallel.map(@bulk, in_threads: @threads) do |element|
+          element.valid?
+        end.all?
+        raise Rimportor::Error::BulkValidation.new("Your bulk is not valid") unless validation_result
+      end
+
+      def execute_callbacks(element, before_or_after)
+        case before_or_after
+          when :before
+            element.run_callbacks(:save) { false }
+          when :after
+            element.run_callbacks(:save) { true }
+        end
+      end
+
+      def import_statement(batch)
+        insert_statement = SqlBuilder.new(batch.first).full_insert_statement
+        result = ::Parallel.map(batch.drop(1), in_threads: @threads) do |element|
+          @adapter.exec_in_pool { SqlBuilder.new(element).partial_insert_statement.gsub('VALUES', '') }
+        end
+        [insert_statement, result]
+      end
+
+      def exec_statement
+        begin
+          run_validations if @validate_bulk
+          run_before_callbacks if @before_callbacks
+          @bulk.each_slice(@batch_size) do |batch|
+            @adapter.exec_insert(import_statement(batch))
+          end
+          run_after_callbacks if @after_callbacks
+          true
+        rescue => e
+          puts "Error importing the bulk. Reason #{e.message}"
+          false
+        end
+      end
+
+    end
+  end
+end

data/lib/rimportor/active_record/sql_builder.rb

@@ -0,0 +1,43 @@
+module Rimportor
+  module ActiveRecord
+    class SqlBuilder
+
+      def initialize(model)
+        @model = model
+        set_timestamps
+      end
+
+      def full_insert_statement
+        insert_manager.tap do |im|
+          im.insert(arel_for_create)
+        end.to_sql
+      end
+
+      def partial_insert_statement
+        insert_manager.insert(arel_for_create).to_sql
+      end
+
+      def arel_for_create
+        @model.send(:arel_attributes_with_values_for_create, @model.attribute_names)
+      end
+
+      def insert_manager
+        @model.class.arel_table.create_insert
+      end
+
+      def set_timestamps
+        set_created_at
+        set_updated_at
+      end
+
+      def set_created_at
+        @model.updated_at = Time.zone.now if @model.respond_to? :updated_at
+      end
+
+      def set_updated_at
+        @model.created_at = Time.zone.now if @model.respond_to? :created_at
+      end
+
+    end
+  end
+end

data/lib/rimportor/error/bulk_validation.rb

@@ -0,0 +1,6 @@
+module Rimportor
+  module Error
+    class BulkValidation < StandardError
+    end
+  end
+end

data/lib/rimportor/error/invalid_adapter.rb

@@ -0,0 +1,6 @@
+module Rimportor
+  module Error
+    class InvalidAdapter < StandardError
+    end
+  end
+end

data/lib/rimportor/plugin.rb

@@ -0,0 +1,28 @@
+module Rimportor
+  module Plugin
+    extend ActiveSupport::Concern
+
+    included do
+    end
+
+    module ClassMethods
+      def rimport(records, options = {})
+        ::Rimportor::ActiveRecord::Import.new(records, self.current_adapter, options).exec_statement
+      end
+
+      def current_adapter
+        load_adapter(::ActiveRecord::Base.connection_config[:adapter])
+      end
+
+      def load_adapter(adapter_name)
+        begin
+          ::Rimportor::ActiveRecord::Adapter.const_get(adapter_name.to_s.camelize).new
+        rescue => e
+          raise ::Rimportor::Error::InvalidAdapter.new("Invalid adapter. Reason #{e}")
+        end
+      end
+
+    end
+
+  end
+end

data/lib/rimportor/util/connection.rb

@@ -0,0 +1,13 @@
+module Rimportor
+  module Util
+    class Connection
+
+      def self.in_pool
+        ::ActiveRecord::Base.connection_pool.with_connection do |connection|
+          yield(connection)
+        end
+      end
+
+    end
+  end
+end

data/lib/rimportor/version.rb

@@ -0,0 +1,3 @@
+module Rimportor
+  VERSION = "0.3"
+end

data/lib/rimportor.rb

@@ -0,0 +1,29 @@
+require 'rimportor/active_record/sql_builder'
+require 'rimportor/active_record/import'
+require 'rimportor/plugin'
+require 'rimportor/error/bulk_validation'
+require 'rimportor/error/invalid_adapter'
+require 'rimportor/active_record/adapter/mysql2'
+require 'rimportor/util/connection'
+require 'generators/install_generator'
+
+module Rimportor
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration) if block_given?
+  end
+
+  class Configuration
+    attr_accessor :threads
+
+    def initialize
+      @threads = 4
+    end
+  end
+end
+
+ActiveRecord::Base.send :include, Rimportor::Plugin

data/lib/templates/rimportor.rb

@@ -0,0 +1,7 @@
+Rimportor.configure do |config|
+  # Configure how many threads rimportor should use for importing.
+  # Consider that rimportor will use threads not only for building the statement
+  # but also for running validations for your bulk.
+  # The default value are 4 threads
+  # config.threads = 4
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: rimportor
+version: !ruby/object:Gem::Version
+  version: '0.3'
+platform: ruby
+authors:
+- Erwin Schens
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-11-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: Fast, modern and concurrent bulk import for ruby on rails.
+email:
+- erwin.schens@qurasoft.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- lib/generators/install_generator.rb
+- lib/rimportor.rb
+- lib/rimportor/active_record/adapter/mysql2.rb
+- lib/rimportor/active_record/import.rb
+- lib/rimportor/active_record/sql_builder.rb
+- lib/rimportor/error/bulk_validation.rb
+- lib/rimportor/error/invalid_adapter.rb
+- lib/rimportor/plugin.rb
+- lib/rimportor/util/connection.rb
+- lib/rimportor/version.rb
+- lib/templates/rimportor.rb
+homepage: https://github.com/ndea/rimportor
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.8
+signing_key: 
+specification_version: 4
+summary: Fast, modern and concurrent bulk import for ruby on rails.
+test_files: []