postgres_upsert 4.0.0 → 5.0.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OGQ4YTc4MTFmYzBjYzE1MWY3ODI3NGQ5Yjc5Njk4Mjk2MGJlZjdhNg==
+    NmI3N2Q0NmE4M2E3NjcxNzQxOTA2MDQ5NzAzMTZlMzk1Y2QzNmNlZQ==
   data.tar.gz: !binary |-
-    ZmNhNzkxNDg4NzhlOWFhYjRjMWNkZDI5ZTMyMGQwMjZlMmM2ZjI4NQ==
+    MjEwODZlMTFlOGIwNTk2NGE3Y2FjMThhYjM0ZDU1YTZkOTAzOWFlMA==
 SHA512:
   metadata.gz: !binary |-
-    ODU0N2IzZmM4ZDc1ODgzYzZmMDM1YzEyNTgwN2Y3MWUyMjI0ZmIxNjIzMjEy
-    M2YyOGFlZjRiM2MzMjcwMTU2MzZjM2IyNWExNWQ3YTFiMzk4ZDIzYWY0MzZi
-    ZTNmNGUwMDAxZDQxNGUwYjI4MTk0MWU3OWYxZDYwMzE5YmU0YjE=
+    MjQ5NTI0MzAyOWM0NTU3MmI2ZjBmYjMzNWJkZTcyYTdiOWNjNTk4MDYxMTdi
+    NDY3ZmQwNjRiNjE0YjE0ZDgyYzU0N2Q1NzhhZTcxNjRmZThiNjUyZTJkMTkw
+    NjY0NDI2ODU1OWYyNjA3ZmVlY2EwODRiMzBmYTNhYzQxODQ2ODk=
   data.tar.gz: !binary |-
-    ZTVmMDBkMmFjZDRkNzIwZjI0MDNhNWRkZjlkMjg4YWU2YTI1MGM4NDVjMWY4
-    MzI2NDE2MzMwOWQzY2Y3N2UwMjYxZDY1YzE4YTJkMDc2ZDg1M2MyZGVlN2Uz
-    NzA0YjQ1YWUyNTBjMDhmNDNmZTNmYTNkZjE4ZmY0YTk2YWYxNzY=
+    ZGIyNzNiZGUxOTNmZDVlNTQxMWRkNjk4OWYxZWUyNTQ2OTkzNGZlYzBjOGVj
+    NmY2NDNlZmZmNWYzZTEwZTEwZWEzMjAyYzE3ZGQyNjUyY2NjY2Q4NGFjODgw
+    MzY1ZjZlNjUwZjU4MmY3NDMwOGViMDhlNjJmZThhN2MyNjg3NGY=

data/.gitignore

@@ -9,6 +9,8 @@
 /test/version_tmp/
 /tmp/
 /log
+.DS_Store
+*/.Ds_Store
 
 ## Specific to RubyMotion:
 .dat*

data/.travis.yml

@@ -1,9 +1,5 @@
 language: ruby
 rvm:
   - 2.1.0
-  - jruby-18mode
-  - jruby-19mode
   - rbx-2
   - ruby-head
-  - jruby-head
-  - ree

data/Gemfile.lock

@@ -1,9 +1,9 @@
 PATH
   remote: .
   specs:
-    postgres_upsert (4.0.0)
+    postgres_upsert (5.0.0)
       activerecord (>= 3.0.0)
-      pg (~> 0.17.0)
+      pg (>= 0.17.0)
       rails (>= 3.0.0)
 
 GEM
@@ -49,9 +49,8 @@ GEM
     coderay (1.1.0)
     diff-lcs (1.1.3)
     erubis (2.7.0)
-    globalid (0.3.3)
+    globalid (0.3.5)
       activesupport (>= 4.1.0)
-    hike (1.2.3)
     i18n (0.7.0)
     json (1.8.2)
     loofah (2.0.1)
@@ -59,13 +58,12 @@ GEM
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
     method_source (0.8.2)
-    mime-types (2.4.3)
+    mime-types (2.6.1)
     mini_portile (0.6.2)
     minitest (5.5.1)
-    multi_json (1.10.1)
     nokogiri (1.6.6.2)
       mini_portile (~> 0.6.0)
-    pg (0.17.1)
+    pg (0.18.2)
     pry (0.10.1)
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
@@ -120,18 +118,14 @@ GEM
       rspec-expectations (~> 2.99.0)
       rspec-mocks (~> 2.99.0)
     slop (3.6.0)
-    sprockets (2.12.3)
-      hike (~> 1.2)
-      multi_json (~> 1.0)
+    sprockets (3.2.0)
       rack (~> 1.0)
-      tilt (~> 1.1, != 1.3.0)
-    sprockets-rails (2.2.4)
+    sprockets-rails (2.3.2)
       actionpack (>= 3.0)
       activesupport (>= 3.0)
       sprockets (>= 2.8, < 4.0)
     thor (0.19.1)
     thread_safe (0.3.4)
-    tilt (1.4.1)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
 

data/README.md

@@ -28,7 +28,7 @@ PostgresUpsert.write <class_or_table_name>, <io_object_or_file_path>[, options]
 options:
 - :delimiter - the string to use to delimit fields from the source data.  Default is ","
 - :header => specifies if the file/io source contains a header row.  Either :header option must be true, or :columns list must be passed.  Default true
-- :key_column => the primary key or unique key column on your destination table, used to distinguish new records from existing records.  Default is the primary_key of your destination table/model.
+- :unique_key => the primary key or unique key column (or composite key columns) on your destination table, used to distinguish new records from existing records.  Default is the primary_key of your destination table/model.
 - :update_only => when true, postgres_upsert will ONLY update existing records, and not insert new.  Default is false.
 
 ## Examples
@@ -66,16 +66,23 @@ currently postgres_upsert detects and manages the default rails timestamp column
 * records that are in the destination table but not the source will not have their timestamps changed.
 
 
-### Overriding the key_column
+### Overriding the unique_key
 
-By default postgres_upsert uses the primary key on your ActiveRecord table to determine if each record should be inserted or updated.  You can override the column using the :key_field option:
+By default postgres_upsert uses the primary key on your ActiveRecord table to determine if each record should be inserted or updated.  You can override the column using the :unique_key option:
 
 ```ruby
-PostgresUpsert.write User "/tmp/users.csv", :key_column => ["external_twitter_id"]
+PostgresUpsert.write User "/tmp/users.csv", :unique_key => ["external_twitter_id"]
 ```
 
 obviously, the field you pass must be a unique key in your database (this is not enforced at the moment, but will be)
 
+If your source data does not contain the primary key, or an individual unique key, you can pass multiple columns in the unique_key option:
+
+```ruby
+  PostgresUpsert.write User "/tmp/users.csv", :unique_key => ["state_id_numer", "state_name"]
+```
+As long as combined columns represent a unique value, row, we can successfully upsert.
+
 passing :update_only => true will ensure that no new records are created, but records will be updated.
 
 ### Insert/Update Counts
@@ -124,7 +131,7 @@ def insert_smart
         end
       end
       io = StringIO.new(csv_string)
-      PostgresUpsert.write User io, key_column: "email"
+      PostgresUpsert.write User io, unique_key: "email"
     end
     puts time
 end

data/db/migrate/20150710162236_create_composite_models_table.rb

@@ -0,0 +1,9 @@
+class CreateCompositeModelsTable < ActiveRecord::Migration
+  def change
+    create_table :composite_key_models do |t|
+      t.integer :comp_key_1
+      t.integer :comp_key_2
+      t.string :data
+    end
+  end
+end

data/db/schema.rb

@@ -11,18 +11,33 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20150214192135) do
+ActiveRecord::Schema.define(version: 20150710162236) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
-# Could not dump table "reserved_word_models" because of following NoMethodError
-#   undefined method `[]' for nil:NilClass
+  create_table "composite_key_models", force: :cascade do |t|
+    t.integer "comp_key_1"
+    t.integer "comp_key_2"
+    t.string  "data"
+  end
 
-# Could not dump table "test_models" because of following NoMethodError
-#   undefined method `[]' for nil:NilClass
+  create_table "reserved_word_models", force: :cascade do |t|
+    t.string "select", limit: 255
+    t.string "group",  limit: 255
+  end
 
-# Could not dump table "three_columns" because of following NoMethodError
-#   undefined method `[]' for nil:NilClass
+  create_table "test_models", force: :cascade do |t|
+    t.string   "data",       limit: 255
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
+
+  create_table "three_columns", force: :cascade do |t|
+    t.string   "data",       limit: 255
+    t.string   "extra",      limit: 255
+    t.datetime "created_at"
+    t.datetime "updated_at"
+  end
 
 end

data/lib/postgres_upsert/result.rb

@@ -2,9 +2,22 @@ module PostgresUpsert
   class Result
     attr_reader :inserted, :updated
 
-    def initialize(insert_result, update_result)
+    def initialize(insert_result, update_result, copy_result)
       @inserted = insert_result ? insert_result.cmd_tuples : 0
       @updated = update_result ? update_result.cmd_tuples : 0  
+      @copied = copy_result ? copy_result.cmd_tuples : 0  
+    end
+
+    def changed_rows
+      @inserted + @updated
+    end
+
+    def copied_rows
+      @copied
+    end
+
+    def updated_rows
+      @updated
     end
   end
 end

data/lib/postgres_upsert/table_writer.rb

@@ -10,6 +10,10 @@ module PostgresUpsert
 
   private
 
+    def database_connection
+      ActiveRecord::Base.connection
+    end
+
     def primary_key
       @primary_key ||= begin
         query = <<-sql

data/lib/postgres_upsert/writer.rb

@@ -4,17 +4,18 @@ module PostgresUpsert
     def initialize(klass, source, options = {})
       @klass = klass
       @options = options.reverse_merge({
-        :delimiter => ",", 
-        :header => true, 
-        :key_column => primary_key,
+        :delimiter => ",",
+        :header => true,
+        :unique_key => [primary_key],
         :update_only => false})
+      @options[:unique_key] = Array.wrap(@options[:unique_key])
       @source = source.instance_of?(String) ? File.open(source, 'r') : source
       @columns_list = get_columns
       generate_temp_table_name
     end
 
     def write
-      if @columns_list.empty? 
+      if @columns_list.empty?
         raise "Either the :columns option or :header => true are required"
       end
 
@@ -24,21 +25,37 @@ module PostgresUpsert
       columns_string = columns_string_for_copy
       create_temp_table
 
-      ActiveRecord::Base.connection.raw_connection.copy_data %{COPY #{copy_table} #{columns_string} FROM STDIN #{csv_options}} do
+      @copy_result = database_connection.raw_connection.copy_data %{COPY #{copy_table} #{columns_string} FROM STDIN #{csv_options}} do
 
         while line = @source.gets do
           next if line.strip.size == 0
-          ActiveRecord::Base.connection.raw_connection.put_copy_data line
+          database_connection.raw_connection.put_copy_data line
         end
       end
 
       upsert_from_temp_table
       drop_temp_table
-      PostgresUpsert::Result.new(@insert_result, @update_result)
+
+      summarize_results
     end
 
   private
 
+    def database_connection
+      @klass.connection
+    end
+
+    def summarize_results
+      result = PostgresUpsert::Result.new(@insert_result, @update_result, @copy_result)
+      expected_rows = @options[:update_only] ? result.updated_rows : result.copied_rows
+      
+      if result.changed_rows != expected_rows
+        raise "#{expected_rows} rows were copied, but #{result.changed_rows} were upserted to destination table.  Check to make sure your key is unique."
+      end
+
+      return result
+    end
+
     def primary_key
       @klass.primary_key
     end
@@ -93,7 +110,9 @@ module PostgresUpsert
 
     def select_string_for_create
       columns = @columns_list.map(&:to_sym)
-      columns << @options[:key_column].to_sym unless columns.include?(@options[:key_column].to_sym)
+      @options[:unique_key].each do |key_component|
+        columns << key_component.to_sym unless columns.include?(key_component.to_sym)
+      end
       get_columns_string(columns)
     end
 
@@ -112,12 +131,12 @@ module PostgresUpsert
     end
 
     def update_from_temp_table
-      @update_result = ActiveRecord::Base.connection.execute <<-SQL
+      @update_result = database_connection.execute <<-SQL
         UPDATE #{quoted_table_name} AS d
           #{update_set_clause}
           FROM #{@temp_table_name} as t
-          WHERE t.#{@options[:key_column]} = d.#{@options[:key_column]}
-          AND d.#{@options[:key_column]} IS NOT NULL;
+          WHERE #{unique_key_select("t", "d")}
+          AND #{unique_key_present("d")}
       SQL
     end
 
@@ -132,39 +151,48 @@ module PostgresUpsert
     def insert_from_temp_table
       columns_string = columns_string_for_insert
       select_string = select_string_for_insert
-      @insert_result = ActiveRecord::Base.connection.execute <<-SQL
+      @insert_result = database_connection.execute <<-SQL
         INSERT INTO #{quoted_table_name} (#{columns_string})
           SELECT #{select_string}
           FROM #{@temp_table_name} as t
-          WHERE NOT EXISTS 
-            (SELECT 1 
-                  FROM #{quoted_table_name} as d 
-                  WHERE d.#{@options[:key_column]} = t.#{@options[:key_column]})
-          AND t.#{@options[:key_column]} IS NOT NULL;
+          WHERE NOT EXISTS
+            (SELECT 1
+                  FROM #{quoted_table_name} as d
+                  WHERE #{unique_key_select("t", "d")});
       SQL
     end
 
+    def unique_key_select(source, dest)
+      @options[:unique_key].map {|field| "#{source}.#{field} = #{dest}.#{field}"}.join(' AND ')
+    end
+
+    def unique_key_present(source)
+      @options[:unique_key].map {|field| "#{source}.#{field} IS NOT NULL"}.join(' AND ')
+    end
+
     def create_temp_table
       columns_string = select_string_for_create
       verify_temp_has_key
-      ActiveRecord::Base.connection.execute <<-SQL
+      database_connection.execute <<-SQL
         SET client_min_messages=WARNING;
         DROP TABLE IF EXISTS #{@temp_table_name};
 
-        CREATE TEMP TABLE #{@temp_table_name} 
+        CREATE TEMP TABLE #{@temp_table_name}
           AS SELECT #{columns_string} FROM #{quoted_table_name} WHERE 0 = 1;
       SQL
     end
 
     def verify_temp_has_key
-      unless @columns_list.include?(@options[:key_column].to_s)
-        raise "Expected a unique column '#{@options[:key_column]}' but the source data does not include this column.  Update the :columns list or explicitly set the :key_column option.}"
+      @options[:unique_key].each do |key_component|
+        unless @columns_list.include?(key_component.to_s)
+          raise "Expected a unique column '#{key_component}' but the source data does not include this column.  Update the :columns list or explicitly set the unique_key option.}"
+       end
       end
     end
 
     def drop_temp_table
-      ActiveRecord::Base.connection.execute <<-SQL
-        DROP TABLE #{@temp_table_name} 
+      database_connection.execute <<-SQL
+        DROP TABLE #{@temp_table_name}
       SQL
     end
   end

data/postgres_upsert.gemspec

@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name = "postgres_upsert"
-  s.version = "4.0.0"
+  s.version = "5.0.0"
 
   s.platform    = Gem::Platform::RUBY
   s.required_ruby_version     = ">= 1.8.7"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
   s.summary = "A rubygem that integrates with ActiveRecord to insert/update large data sets into the database efficiently"
 
-  s.add_dependency "pg", '~> 0.17.0'
+  s.add_dependency "pg", '>= 0.17.0'
   s.add_dependency "activerecord", '>= 3.0.0'
   s.add_dependency "rails", '>= 3.0.0'
   s.add_development_dependency "bundler"

data/spec/composite_key_spec.rb

@@ -0,0 +1,56 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "pg_upsert from file with CSV format" do
+  before(:each) do
+    ActiveRecord::Base.connection.execute %{
+      TRUNCATE TABLE composite_key_models;
+      SELECT setval('composite_key_models_id_seq', 1, false);
+    }
+  end
+
+  before do
+    DateTime.stub_chain(:now, :utc).and_return (DateTime.parse("2012-01-01").utc)
+  end
+
+  def timestamp
+    DateTime.now.utc
+  end
+
+  context 'composite_key_support' do
+    it 'inserts records if the passed match composite key doesnt exist' do
+      file = File.open(File.expand_path('spec/fixtures/composite_key_with_header.csv'), 'r')
+
+      PostgresUpsert.write(CompositeKeyModel, file, :unique_key => ["comp_key_1", "comp_key_2"])
+      expect(
+        CompositeKeyModel.last.attributes
+      ).to include("data" => "test data 2")
+    end
+
+    it 'updates records if the passed composite key exists' do
+      file = File.open(File.expand_path('spec/fixtures/composite_key_with_header.csv'), 'r')
+      existing = CompositeKeyModel.create(comp_key_1: 2, comp_key_2:3, data: "old stuff")
+
+      PostgresUpsert.write(CompositeKeyModel, file, :unique_key => ["comp_key_1", "comp_key_2"])
+
+      expect(
+        CompositeKeyModel.find_by({comp_key_1: 2, comp_key_2:3}).attributes
+      ).to include("data" => "test data 2")
+
+      expect(
+        CompositeKeyModel.find_by({comp_key_1: 1, comp_key_2:2}).attributes
+      ).to include("data" => "test data 1")
+    end
+
+    it 'fails if composite keys are not unique.' do
+      file = File.open(File.expand_path('spec/fixtures/composite_nonkey_with_header.csv'), 'r')
+      existing = CompositeKeyModel.create(comp_key_1: 1, comp_key_2:2, data: "old stuff")
+
+       expect{
+        PostgresUpsert.write(CompositeKeyModel, file, :unique_key => ["comp_key_1", "comp_key_2"])
+      }.to raise_error(/Check to make sure your key is unique/)
+    end
+    
+  end
+
+
+end

data/spec/fixtures/composite_key_model.rb

@@ -0,0 +1,4 @@
+require 'postgres_upsert'
+
+class CompositeKeyModel < ActiveRecord::Base
+end

data/spec/fixtures/composite_key_with_header.csv

@@ -0,0 +1,3 @@
+comp_key_1,comp_key_2,data
+1,2,test data 1
+2,3,test data 2

data/spec/fixtures/composite_nonkey_with_header.csv

@@ -0,0 +1,3 @@
+comp_key_1,comp_key_2,data
+1,2,test data 1
+1,2,test data 4

data/spec/pg_upsert_csv_spec.rb

@@ -173,7 +173,7 @@ describe "pg_upsert from file with CSV format" do
       three_col = ThreeColumn.create(id: 1, data: "old stuff", extra: "neva change!")
       file = File.open(File.expand_path('spec/fixtures/no_id.csv'), 'r')
 
-      PostgresUpsert.write(ThreeColumn, file, :key_column => "data")
+      PostgresUpsert.write(ThreeColumn, file, :unique_key => "data")
       expect(
         three_col.reload.extra
       ).to eq("ABC: Always Be Changing.")
@@ -182,7 +182,7 @@ describe "pg_upsert from file with CSV format" do
     it 'inserts records if the passed match column doesnt exist' do
       file = File.open(File.expand_path('spec/fixtures/no_id.csv'), 'r')
 
-      PostgresUpsert.write(ThreeColumn, file, :key_column => "data")
+      PostgresUpsert.write(ThreeColumn, file, :unique_key => "data")
       expect(
         ThreeColumn.last.attributes
       ).to include("data" => "old stuff", "extra" => "ABC: Always Be Changing.")
@@ -191,7 +191,7 @@ describe "pg_upsert from file with CSV format" do
     it 'allows key column to be a string or symbol' do
       file = File.open(File.expand_path('spec/fixtures/no_id.csv'), 'r')
 
-      PostgresUpsert.write(ThreeColumn, file, :header => true, :key_column => :data)
+      PostgresUpsert.write(ThreeColumn, file, :header => true, :unique_key => :data)
       expect(
         ThreeColumn.last.attributes
       ).to include("data" => "old stuff", "extra" => "ABC: Always Be Changing.")
@@ -247,7 +247,7 @@ describe "pg_upsert from file with CSV format" do
 
     it "should still report results" do
       TestModel.create(data: "test data 1")
-      result = result = PostgresUpsert.write TestModel.table_name, File.expand_path('spec/fixtures/tab_with_two_lines.csv'), :delimiter => "\t"
+      result = PostgresUpsert.write TestModel.table_name, File.expand_path('spec/fixtures/tab_with_two_lines.csv'), :delimiter => "\t"
 
       expect(
         result.updated
@@ -258,7 +258,6 @@ describe "pg_upsert from file with CSV format" do
       ).to eq(1)
     end
 
-
   end
 end
 

data/spec/spec_helper.rb

@@ -4,6 +4,7 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 require 'fixtures/test_model'
 require 'fixtures/three_column'
 require 'fixtures/reserved_word_model'
+require 'fixtures/composite_key_model'
 require 'rspec'
 require 'rspec/rails'
 require 'rspec/autorun'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postgres_upsert
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 5.0.0
 platform: ruby
 authors:
 - Steve Mitchell
@@ -14,14 +14,14 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: 0.17.0
 - !ruby/object:Gem::Dependency
@@ -139,6 +139,7 @@ files:
 - config/routes.rb
 - config/secrets.yml
 - db/migrate/20150214192135_create_test_tables.rb
+- db/migrate/20150710162236_create_composite_models_table.rb
 - db/schema.rb
 - db/seeds.rb
 - lib/postgres_upsert.rb
@@ -146,10 +147,14 @@ files:
 - lib/postgres_upsert/table_writer.rb
 - lib/postgres_upsert/writer.rb
 - postgres_upsert.gemspec
+- spec/composite_key_spec.rb
 - spec/fixtures/comma_with_header.csv
 - spec/fixtures/comma_with_header_and_comma_values.csv
 - spec/fixtures/comma_with_header_and_unquoted_comma.csv
 - spec/fixtures/comma_without_header.csv
+- spec/fixtures/composite_key_model.rb
+- spec/fixtures/composite_key_with_header.csv
+- spec/fixtures/composite_nonkey_with_header.csv
 - spec/fixtures/no_id.csv
 - spec/fixtures/reserved_word_model.rb
 - spec/fixtures/reserved_words.csv
@@ -190,10 +195,14 @@ specification_version: 4
 summary: A rubygem that integrates with ActiveRecord to insert/update large data sets
   into the database efficiently
 test_files:
+- spec/composite_key_spec.rb
 - spec/fixtures/comma_with_header.csv
 - spec/fixtures/comma_with_header_and_comma_values.csv
 - spec/fixtures/comma_with_header_and_unquoted_comma.csv
 - spec/fixtures/comma_without_header.csv
+- spec/fixtures/composite_key_model.rb
+- spec/fixtures/composite_key_with_header.csv
+- spec/fixtures/composite_nonkey_with_header.csv
 - spec/fixtures/no_id.csv
 - spec/fixtures/reserved_word_model.rb
 - spec/fixtures/reserved_words.csv