fast_inserter 0.1.6 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fe09e32230a3dc433d82f2afe7329cfe6388c46d
-  data.tar.gz: ea29978534d0cb33c67baafa43fbcd1f8d02735a
+SHA256:
+  metadata.gz: b34b8efaffa71fa07824869fc4786cfbd075c878ba99c96989846bf0e3a5c377
+  data.tar.gz: e688e45fe0316029dab434a56ff1ad5f2c1cdfb7644995d2f4fcc3762ecbf5a8
 SHA512:
-  metadata.gz: 03556b9df683ce5c6bce7797af5d6754a001167aeae82bd4edc43b2ed2eb0fc21a23c15f32fb740a1fe884a846660ccebda1cc514dd1c048fc6b619b0470ca61
-  data.tar.gz: 389035369e1874a8f522a0c5dbd0be6ddc6e9f16dbf6088101021e05def715d5c204dc49dacb278e8405f4a328c72a7739c472bd3698ada9d233140ed826c0a2
+  metadata.gz: aa47c991faaf1170779cac718a85a1d89a28b393feae40014296100281d1ecab918dba51ca64733dc8c193ac7784b2fa5866b9a3642f82219c9bf58007427f4d
+  data.tar.gz: 73b3b9ec678d4588fac02f9477aac000391b34f31d072194be4690798d541823fcccdfda56ac79b854f089e247d2a33377cfc25707384e75594405419a916022

data/.github/workflows/ruby.yml

@@ -0,0 +1,55 @@
+name: Ruby
+
+on: push
+
+jobs:
+  build-and-test-job:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        ruby-version: ['2.6.10', '2.7.8', '3.0.6', '3.1.4', '3.2.2', 'ruby-head']
+        database: [sqlite, pg, mysql]
+    
+    # Postgres has to be explicitly included. The runner environment already has mysql installed by default.
+    services:
+      postgres:
+        # Pin to this commit: version 14.0
+        image: postgres@sha256:2b8a60d4ae4b3cf9dc5ae823700f699fdabdb0d03901dd480c9a410ca72c4571
+        env:
+          # Password is required. Postgres won't start without it.
+          POSTGRES_PASSWORD: postgres_password
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      # CAUTION - the DB env var is also used by the test setup code. Don't change the name of it!
+      DB: ${{ matrix.database }}
+      RV: ${{ matrix.ruby-version }}
+      PGPASSWORD: postgres_password
+
+    steps:
+      - name: Init
+        run: echo "Using Ruby $RV and '$DB' as the database."
+      - name: Checkout
+        # Pin to this commit: v2
+        uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
+      - name: Set up Database
+        run: |
+          if [ "$DB" = "pg" ]; then psql -h localhost -c 'DROP DATABASE IF EXISTS fast_inserter;' -U postgres; fi
+          if [ "$DB" = "pg" ]; then psql -h localhost -c 'create database fast_inserter;' -U postgres; fi
+          if [ "$DB" = "mysql" ]; then sudo systemctl start mysql.service; fi
+          if [ "$DB" = "mysql" ]; then mysql -e 'create database IF NOT EXISTS fast_inserter;' -uroot -proot; fi
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true # runs 'bundle install' and caches installed gems automatically
+      - name: Run tests
+        # The test setup code reads the DB env var to determine which database to use.
+        run: bundle exec rake

data/.ruby-version

@@ -1 +1 @@
-2.3.0
+3.0.6

data/CHANGELOG.md

@@ -1,4 +1,24 @@
-## Unreleased ##
+## 2.0.0 (April 20, 2023) ##
+
+* Drop Ruby version 2.4 and 2.5 support
+* Add Ruby versions 3.0.6, 3.1.4, 3.2.2
+* Update development Ruby version to 3.0.6
+* Update type casting in preparation for rails 7
+* Account for MySQL type casting in ActiveRecord 7
+
+    *Amy Lin, Tim Kelly*
+
+## 1.0.0.pre (January 7, 2019) ##
+
+*   Use database IN clause when checking for existing records.
+
+    *Jordon Dornbos*
+
+## 0.1.6 (July 3, 2018) ##
+
+*   Fix check_for_existing with multiple variable columns.
+
+    *Josh Warfield*
 
 *   Remove support for Ruby 2.2. Add support for Ruby 2.4 and 2.5.
 

data/README.md

@@ -1,7 +1,7 @@
 # FastInserter
 
+[![Build Status](https://github.com/joinhandshake/fast_inserter/actions/workflows/ruby.yml/badge.svg)](https://github.com/joinhandshake/fast_inserter/actions)
 [![Gem Version](https://badge.fury.io/rb/fast_inserter.svg)](https://badge.fury.io/rb/fast_inserter)
-[![Build Status](https://travis-ci.org/joinhandshake/fast_inserter.svg?branch=master)](https://travis-ci.org/joinhandshake/fast_inserter)
 
 Use raw SQL to insert database records in bulk, fast. Supports uniqueness constraints, timestamps, and checking for existing records.
 
@@ -51,9 +51,9 @@ params = {
   options: {
     timestamps: true,
     unique: true,
-    check_for_existing: true,
-    group_size: 2_000
+    check_for_existing: true
   },
+  group_size: 2_000,
   variable_column: 'user_id',
   values: user_ids
 }
@@ -89,7 +89,7 @@ Queries the table for any values which already exist and removes them from the v
 
 ### group_size
 
-Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 2,000.
+Insertions will be broken up into batches. This specifies the number of records you want to insert per batch. Default is 1,000.
 
 ### variable_column
 

data/fast_inserter.gemspec

@@ -21,16 +21,16 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'activerecord', '>= 4.1.0'
 
-  spec.required_ruby_version = ">= 2.2.0"
+  spec.required_ruby_version = ">= 2.3.8"
 
   spec.add_development_dependency "bundler"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake"
   spec.add_development_dependency "rspec"
   spec.add_development_dependency "database_cleaner"
 
   case ENV['DB']
   when "mysql"; spec.add_development_dependency "mysql2"
-  when "sqlite"; spec.add_development_dependency "sqlite3"
+  when "sqlite"; spec.add_development_dependency "sqlite3", ">= 1.4"
   when "pg"; spec.add_development_dependency "pg"
   else spec.add_development_dependency "sqlite3" # Default
   end

data/lib/fast_inserter/fast_inserter_base.rb

@@ -1,3 +1,5 @@
+require 'enumerator'
+
 # Highly based off of https://github.com/sportngin/m2m_fast_insert
 # Unfortunately, that gem was not up to date for rails 4.
 #
@@ -18,8 +20,7 @@
 #     options: {
 #       timestamps: true,
 #       unique: true,
-#       check_for_existing: true,
-#       group_size: 1_000
+#       check_for_existing: true
 #     },
 #     variable_column: 'user_id',
 #     values: user_ids
@@ -39,11 +40,11 @@
 #     A hash representing additional column values to set that you don't want
 #     to include in uniqueness checks or other pre-insertion operations.
 #   group_size: Integer
-#     The number of items you want to insert per batch of records. Default 10_000.
+#     The number of items you want to insert per batch of records. Defaults to 1_000.
 #
 module FastInserter
   class Base
-    DEFAULT_GROUP_SIZE = 2_000
+    DEFAULT_GROUP_SIZE = 1_000
 
     def initialize(params)
       @table_name = params[:table]
@@ -58,7 +59,7 @@ module FastInserter
       all_values = params[:values].map { |value| Array(value) }
       all_values.uniq! if @options[:unique]
       group_size = Integer(params[:group_size] || ENV['FAST_INSERTER_GROUP_SIZE'] || DEFAULT_GROUP_SIZE)
-      @value_groups = all_values.in_groups_of(group_size, false)
+      @value_groups = all_values.each_slice(group_size).to_a
     end
 
     # Iterates through the value groups (which is all values in groups of smaller sizes)
@@ -92,34 +93,70 @@ module FastInserter
       end
     end
 
+    def existing_values_sql(group_of_values)
+      sql = "SELECT #{@variable_columns.join(', ')} FROM #{@table_name} WHERE #{values_hash_to_sql(@static_columns)}"
+
+      if @variable_columns.length > 1
+        group_of_values_sql = group_of_values.map do |group|
+          values_hash = variable_column_values_to_hash(group)
+          "(#{values_hash_to_sql(values_hash)})"
+        end.join(' OR ')
+
+        sql += " AND (#{group_of_values_sql})"
+      else
+        values_to_check = ActiveRecord::Base.send(:sanitize_sql_array, ['?', group_of_values.flatten])
+        sql += " AND #{@variable_columns.first} IN (#{values_to_check})"
+      end
+
+      sql
+    end
+
     # Queries for the existing values for a given group of values
     def existing_values(group_of_values)
-      sql = "SELECT #{@variable_columns.join(', ')} FROM #{@table_name} WHERE #{existing_values_static_columns}"
+      sql = existing_values_sql(group_of_values)
 
       # NOTE: There are more elegant ways to get this field out of the resultset, but each database adaptor returns a different type
       # of result from 'execute(sql)'. Potential classes for 'result' is Array (sqlite), Mysql2::Result (mysql2), PG::Result (pg). Each
       # result can be enumerated into a list of arrays (mysql) or list of hashes (sqlite, pg)
       results = ActiveRecord::Base.connection.execute(sql)
-      existing_values = stringify_values(results)
-
-      # Rather than a giant IN query in the sql statement (which can be bad for database performance),
-      # do the filtering of relevant values here in a ruby select.
-      group_of_values_strings = stringify_values(group_of_values)
-      existing_values & group_of_values_strings
+      stringify_values(results)
     end
 
     def stringify_values(results)
       results.to_a.map do |result|
         if result.is_a?(Hash)
-          @variable_columns.map { |col| ActiveRecord::Base.connection.type_cast(result[col], column_definitions[col]) }
+          @variable_columns.map { |col| type_cast_column(result[col], column_definitions[col]) }
         elsif result.is_a?(Array)
-          result.map.with_index { |val, i| ActiveRecord::Base.connection.type_cast(val, column_definitions[@variable_columns[i]]) }
+          result.map.with_index { |val, i| type_cast_column(val, column_definitions[@variable_columns[i]]) }
         end
       end
     end
 
-    def existing_values_static_columns
-      @static_columns.map do |key, value|
+    # Passing a column to the method type_cast is deprecated 
+    # and will be removed in Rails 7. For now, we can manually 
+    # call lookup_cast_type_from_column and serialize our
+    # values with the returned type.
+    def type_cast_column(value, column = nil)
+      if column
+        type = ActiveRecord::Base.connection.lookup_cast_type_from_column(column)
+        value = type.serialize(value)
+      end
+
+      ActiveRecord::Base.connection.type_cast(value)
+    end
+
+    def variable_column_values_to_hash(values)
+      hash = {}
+
+      @variable_columns.each_with_index do |variable_column, index|
+        hash[variable_column] = values[index]
+      end
+
+      hash
+    end
+
+    def values_hash_to_sql(values)
+      values.map do |key, value|
         if value.nil?
           "#{key} IS NULL"
         else

data/lib/fast_inserter/version.rb

@@ -1,3 +1,3 @@
 module FastInserter
-  VERSION = "0.1.6"
+  VERSION = '2.0.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast_inserter
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 2.0.0
 platform: ruby
 authors:
 - Scott Ringwelski
@@ -9,10 +9,10 @@ authors:
 - Jordon Dornbos
 - Matt Hickman
 - Josh Warfield
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-07-03 00:00:00.000000000 Z
+date: 2023-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -46,16 +46,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -110,10 +110,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -129,7 +129,7 @@ homepage: https://github.com/strydercorp/fast_inserter
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -137,16 +137,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.2.0
+      version: 2.3.8
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.2.33
+signing_key:
 specification_version: 4
 summary: Quickly insert database records in bulk
 test_files: []

data/.travis.yml

@@ -1,19 +0,0 @@
-language: ruby
-sudo: required
-notifications:
-  email: false
-rvm:
-  - 2.3.0
-  - 2.4.4
-  - 2.5.1
-  - ruby-head
-env:
-  matrix:
-    - DB=pg
-    - DB=mysql
-    - DB=sqlite
-before_script:
-  # PG and mysql is simpler.
-  - sh -c "if [ '$DB' = 'pg' ]; then psql -c 'DROP DATABASE IF EXISTS fast_inserter;' -U postgres; fi"
-  - sh -c "if [ '$DB' = 'pg' ]; then psql -c 'create database fast_inserter;' -U postgres; fi"
-  - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'create database IF NOT EXISTS fast_inserter;'; fi"