postgres_upsert 3.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 8b0a956bd60011cf247a58c3031d5157be3c6f7f
-  data.tar.gz: 918a0d163efa3915f8b3d76421ab69e67e2469d8
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    OGQ4YTc4MTFmYzBjYzE1MWY3ODI3NGQ5Yjc5Njk4Mjk2MGJlZjdhNg==
+  data.tar.gz: !binary |-
+    ZmNhNzkxNDg4NzhlOWFhYjRjMWNkZDI5ZTMyMGQwMjZlMmM2ZjI4NQ==
 SHA512:
-  metadata.gz: d1dda03e61a25fb875bbaf8eab001d0f8b780dbcb329f8be1c0aabbb03c4be4bb66c6adf2dbfbf21d3edb55135be2fc01826dcc79259b928b64c7bb1748c0369
-  data.tar.gz: ca59859938a0671ffed159495d10a08328a498500f8c18610c4d50f9d452901b55c5e81e63e62eb35dc8c74b532a713e4f2285db690721be5f4de9b3aed3f8ad
+  metadata.gz: !binary |-
+    ODU0N2IzZmM4ZDc1ODgzYzZmMDM1YzEyNTgwN2Y3MWUyMjI0ZmIxNjIzMjEy
+    M2YyOGFlZjRiM2MzMjcwMTU2MzZjM2IyNWExNWQ3YTFiMzk4ZDIzYWY0MzZi
+    ZTNmNGUwMDAxZDQxNGUwYjI4MTk0MWU3OWYxZDYwMzE5YmU0YjE=
+  data.tar.gz: !binary |-
+    ZTVmMDBkMmFjZDRkNzIwZjI0MDNhNWRkZjlkMjg4YWU2YTI1MGM4NDVjMWY4
+    MzI2NDE2MzMwOWQzY2Y3N2UwMjYxZDY1YzE4YTJkMDc2ZDg1M2MyZGVlN2Uz
+    NzA0YjQ1YWUyNTBjMDhmNDNmZTNmYTNkZjE4ZmY0YTk2YWYxNzY=

data/.gitignore

@@ -8,6 +8,7 @@
 /test/tmp/
 /test/version_tmp/
 /tmp/
+/log
 
 ## Specific to RubyMotion:
 .dat*

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 2.1.0
+  - jruby-18mode
+  - jruby-19mode
+  - rbx-2
+  - ruby-head
+  - jruby-head
+  - ree

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    postgres_upsert (3.0.0)
+    postgres_upsert (4.0.0)
       activerecord (>= 3.0.0)
       pg (~> 0.17.0)
       rails (>= 3.0.0)
@@ -9,44 +9,62 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (4.0.11)
-      actionpack (= 4.0.11)
+    actionmailer (4.2.0)
+      actionpack (= 4.2.0)
+      actionview (= 4.2.0)
+      activejob (= 4.2.0)
       mail (~> 2.5, >= 2.5.4)
-    actionpack (4.0.11)
-      activesupport (= 4.0.11)
-      builder (~> 3.1.0)
-      erubis (~> 2.7.0)
-      rack (~> 1.5.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.0)
+      actionview (= 4.2.0)
+      activesupport (= 4.2.0)
+      rack (~> 1.6.0)
       rack-test (~> 0.6.2)
-    activemodel (4.0.11)
-      activesupport (= 4.0.11)
-      builder (~> 3.1.0)
-    activerecord (4.0.11)
-      activemodel (= 4.0.11)
-      activerecord-deprecated_finders (~> 1.0.2)
-      activesupport (= 4.0.11)
-      arel (~> 4.0.0)
-    activerecord-deprecated_finders (1.0.3)
-    activesupport (4.0.11)
-      i18n (~> 0.6, >= 0.6.9)
-      minitest (~> 4.2)
-      multi_json (~> 1.3)
-      thread_safe (~> 0.1)
-      tzinfo (~> 0.3.37)
-    arel (4.0.2)
-    builder (3.1.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.1)
+    actionview (4.2.0)
+      activesupport (= 4.2.0)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.1)
+    activejob (4.2.0)
+      activesupport (= 4.2.0)
+      globalid (>= 0.3.0)
+    activemodel (4.2.0)
+      activesupport (= 4.2.0)
+      builder (~> 3.1)
+    activerecord (4.2.0)
+      activemodel (= 4.2.0)
+      activesupport (= 4.2.0)
+      arel (~> 6.0)
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.0)
+    builder (3.2.2)
     coderay (1.1.0)
     diff-lcs (1.1.3)
     erubis (2.7.0)
+    globalid (0.3.3)
+      activesupport (>= 4.1.0)
     hike (1.2.3)
-    i18n (0.6.11)
-    json (1.7.6)
+    i18n (0.7.0)
+    json (1.8.2)
+    loofah (2.0.1)
+      nokogiri (>= 1.5.9)
     mail (2.6.3)
       mime-types (>= 1.16, < 3)
     method_source (0.8.2)
     mime-types (2.4.3)
-    minitest (4.7.5)
+    mini_portile (0.6.2)
+    minitest (5.5.1)
     multi_json (1.10.1)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
     pg (0.17.1)
     pry (0.10.1)
       coderay (~> 1.1.0)
@@ -54,47 +72,68 @@ GEM
       slop (~> 3.4)
     pry-rails (0.3.2)
       pry (>= 0.9.10)
-    rack (1.5.2)
-    rack-test (0.6.2)
+    rack (1.6.0)
+    rack-test (0.6.3)
       rack (>= 1.0)
-    rails (4.0.11)
-      actionmailer (= 4.0.11)
-      actionpack (= 4.0.11)
-      activerecord (= 4.0.11)
-      activesupport (= 4.0.11)
+    rails (4.2.0)
+      actionmailer (= 4.2.0)
+      actionpack (= 4.2.0)
+      actionview (= 4.2.0)
+      activejob (= 4.2.0)
+      activemodel (= 4.2.0)
+      activerecord (= 4.2.0)
+      activesupport (= 4.2.0)
       bundler (>= 1.3.0, < 2.0)
-      railties (= 4.0.11)
-      sprockets-rails (~> 2.0)
-    railties (4.0.11)
-      actionpack (= 4.0.11)
-      activesupport (= 4.0.11)
+      railties (= 4.2.0)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.5)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6.0)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.1)
+      loofah (~> 2.0)
+    railties (4.2.0)
+      actionpack (= 4.2.0)
+      activesupport (= 4.2.0)
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
-    rake (10.3.2)
-    rdoc (3.12)
-      json (~> 1.4)
-    rspec (2.12.0)
-      rspec-core (~> 2.12.0)
-      rspec-expectations (~> 2.12.0)
-      rspec-mocks (~> 2.12.0)
-    rspec-core (2.12.2)
-    rspec-expectations (2.12.1)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.12.2)
+    rake (10.4.2)
+    rspec (2.99.0)
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
+    rspec-collection_matchers (1.1.2)
+      rspec-expectations (>= 2.99.0.beta1)
+    rspec-core (2.99.2)
+    rspec-expectations (2.99.2)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.3)
+    rspec-rails (2.99.0)
+      actionpack (>= 3.0)
+      activemodel (>= 3.0)
+      activesupport (>= 3.0)
+      railties (>= 3.0)
+      rspec-collection_matchers
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
     slop (3.6.0)
     sprockets (2.12.3)
       hike (~> 1.2)
       multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    sprockets-rails (2.2.0)
+    sprockets-rails (2.2.4)
       actionpack (>= 3.0)
       activesupport (>= 3.0)
       sprockets (>= 2.8, < 4.0)
     thor (0.19.1)
     thread_safe (0.3.4)
     tilt (1.4.1)
-    tzinfo (0.3.42)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
 
 PLATFORMS
   ruby
@@ -103,5 +142,5 @@ DEPENDENCIES
   bundler
   postgres_upsert!
   pry-rails
-  rdoc
   rspec (~> 2.12)
+  rspec-rails (~> 2.0)

data/README.md

@@ -1,9 +1,11 @@
-# postgres_upsert
+# postgres_upsert [![Build Status](https://travis-ci.org/theSteveMitchell/postgres_upsert.svg?branch=master)](https://travis-ci.org/theSteveMitchell/postgres_upsert)
 
 Allows your  rails app to load data in a very fast way, avoiding calls to ActiveRecord.
 
 Using the PG gem and postgres's powerful COPY command, you can create thousands of rails objects in your db in a single query.
 
+## Compatibility Note
+The master branch requires the 'pg' gem which only supports MRI ruby.  the jruby branch requires 'activerecord-jdbcpostgresql-adapter' which, of course only supports JRuby.  Installation is the same whatever your platform.
 
 ## Install
 
@@ -17,71 +19,135 @@ Run the bundle command
 
 ## Usage
 
-The gem will add the aditiontal class method to ActiveRecord::Base
-
-* pg_upsert io_object_or_file_path, [options]
-
-io_object_or_file_path => is a file path or an io object (StringIO, FileIO, etc.)
+```ruby
+PostgresUpsert.write <class_or_table_name>, <io_object_or_file_path>[, options]
+```
+<class_or_table_name> is either an ActiveRecord::Base subclass, or a string representing the name of a database table.
+<io_object_or_file_path> can be either a string representing a file path, or an io object (StringIO, FileIO, etc.)
 
 options:
-:delimiter - the string to use to delimit fields.  Default is ","
-:format - the format of the file (valid formats are :csv or :binary).  Default is :csv
-:header => specifies if the file/io source contains a header row.  Either :header option must be true, or :columns list must be passed.  Default true
-:key_column => the primary key or unique key column on your ActiveRecord table, used to distinguish new records from existing records.  Default is the primary_key of your ActiveRecord model class.
-:update_only => when true, postgres_upsert will ONLY update existing records, and not insert new.  Default is false.
-
-pg_upsert will allow you to copy data from an arbritary IO object or from a file in the database server (when you pass the path as string).
-Let's first copy from a file in the database server, assuming again that we have a users table and
-that we are in the Rails console:
+- :delimiter - the string to use to delimit fields from the source data.  Default is ","
+- :header => specifies if the file/io source contains a header row.  Either :header option must be true, or :columns list must be passed.  Default true
+- :key_column => the primary key or unique key column on your destination table, used to distinguish new records from existing records.  Default is the primary_key of your destination table/model.
+- :update_only => when true, postgres_upsert will ONLY update existing records, and not insert new.  Default is false.
 
+## Examples
+for these examples let's assume we have a users table and model:
 ```ruby
-User.pg_upsert "/tmp/users.csv"
+class User < ActiveRecord::Base
+```
+In the rails console we can run:
+```ruby
+PostgresUpsert.write User, "/tmp/users.csv"
 ```
 
-This command will use the headers in the CSV file as fields of the target table, so beware to always have a header in the files you want to import.
-If the column names in the CSV header do not match the field names of the target table, you can pass a map in the options parameter.
-
+This command will use the headers in the CSV file as fields of the target table (by default)
+If the CSV file's header does not match the field names of the User class, you can pass a map in the options parameter.
 ```ruby
-User.pg_upsert "/tmp/users.csv", :map => {'name' => 'first_name'}
+PostgresUpsert.write "users", "/tmp/users.csv", :map => {'name' => 'first_name'}
 ```
+The `name` column in the CSV file will be mapped to the `first_name` field in the users table.
+
+postgres_upsert  supports 'merge' operations, which is not yet natively supported in Postgres.  The data can include both new and existing records, and postgres_upsert will handle either update or insert of records appropriately.  Since the Postgres COPY command does not handle this, postgres_upsert accomplishes it using an intermediary temp table.
+
+The merge/upsert happens in 5 steps (assume your data table is called "users")
+* create a temp table named users_temp_123 where "123" is a random int.  In postgres temp tables are only visible to the current database session, so naming conflicts should not be a problem.  We add this random suffix just for additional safety.
+* COPY the data to user_temp
+* issue a query to insert all new records from users_temp_123 into users ("new" records are those records whos primary key does not already exist in the users)
+* issue a query to update all existing records in users with the data in users_temp_123 ("existing" records are those whose primary key already exists in the users table)
+* drop the temp table.
+
+## timestamp columns
+
+currently postgres_upsert detects and manages the default rails timestamp columns `created_at` and `updated_at`.  If these fields exist in your destination table, postgres_upsert will keep these current as expected.  I recommend you do NOT include these fields in your source CSV/IO, as postgres_upsert will not honor them.
 
-In the above example the header name in the CSV file will be mapped to the field called first_name in the users table.
+* newly inserted records get a current timestamp for created_at
+* records existing in the source file/IO will get an update to their updated_at timestamp (even if all fields maintain the same value)
+* records that are in the destination table but not the source will not have their timestamps changed.
 
-To copy a binary formatted data file or IO object you can specify the format as binary
+
+### Overriding the key_column
+
+By default postgres_upsert uses the primary key on your ActiveRecord table to determine if each record should be inserted or updated.  You can override the column using the :key_field option:
 
 ```ruby
-User.pg_upsert "/tmp/users.dat", :format => :binary, :columns => ["id, "name"]
+PostgresUpsert.write User "/tmp/users.csv", :key_column => ["external_twitter_id"]
 ```
 
-Which will generate the following SQL command:
+obviously, the field you pass must be a unique key in your database (this is not enforced at the moment, but will be)
 
-```sql
-COPY users (id, name) FROM '/tmp/users.dat' WITH BINARY
-```
+passing :update_only => true will ensure that no new records are created, but records will be updated.
 
-NOTE: binary files do not include header columns, so passing a :columns array is required for binary files.
+### Insert/Update Counts
+PostgresUpsert with also return a PostgresUpsert::Result object that will tell you how many records were inserted or updated:
 
+```ruby
+User.delete_all
+result = PostgresUpsert.write User "/tmp/users.csv"
+result.inserted 
+# => 10000
+result.updated
+# => 0
+```
 
-pg_upsert  supports 'upsert' or 'merge' operations.  In other words, the data source can contain both new and existing objects, and pg_upsert will handle either case.  Since the Postgres native COPY command does not handle updating existing records, pg_upsert accomplishes update and insert using an intermediary temp table:
+### Huge Caveat!
+Since postgres_upsert does not use validations or even instantiate rails objects, you can get invalid data if you're not careful.  Postgres upsert assumes that your source data is minimally cleaned up, and will not tell you if any data is invalid based on rails model rules.  It will, of course raise an error if data does not conform to your database constraints.
 
-This merge/upsert happend in 5 steps (assume your data table is called "users")
-* create a temp table named users_temp_### where "###" is a random number.  In postgres temp tables are only visible to the current database session, so naming conflicts should not be a problem.
-* COPY the data to user_temp
-* issue a query to insert all new records from users_temp_### into users (newness is determined by the presence of the primary key in the users table)
-* issue a query to update all records in users with the data in users_temp_### (matching on primary key)
-* drop the temp table.
+### Benchmarks!
 
-### overriding the key_column
+Given a User model, (validates presence of email and paassword)
+```console
+2.1.3 :008 > User
+ => User(id: integer, email: string, password: string, created_at: datetime, updated_at: datetime) 
+```
 
-By default pg_upsert uses the primary key on your ActiveRecord table to determine if each record should be inserted or updated.  You can override the column using the :key_field option:
+And the following railsy code to create 10,000 users:
+```ruby
+def insert_dumb
+    time = Benchmark.measure do
+      (1..10000).each do |n|
+        User.create!(:email => "number#{n}@postgres.up", :password => "#{(n-5..n).to_a.join('')}")
+      end
+    end
+  puts time
+end
+```
 
+Compared to the following code using Postgres_upsert:
 ```ruby
-User.pg_upsert "/tmp/users.dat", :format => :binary, :key_column => ["external_twitter_id"]
+def insert_smart
+    time = Benchmark.measure do
+      csv_string = CSV.generate do |csv|
+        csv << %w(email password)
+        (1..10000).each do |n|
+          csv << ["number#{n}@postgres.up", "#{(n-5..n).to_a.join('')}"]
+        end
+      end
+      io = StringIO.new(csv_string)
+      PostgresUpsert.write User io, key_column: "email"
+    end
+    puts time
+end
 ```
 
-obviously, the field you pass must be a unique key in your database (this is not enforced at the moment, but will be)
+let's compare!
+
+```console
+2.1.3 :002 > insert_dumb
+   #...snip  ~30k lines of output :( (10k queries, each wrapped in a transaction)
+   (0.3ms)  COMMIT
+26.639246
+2.1.3 :004 > User.delete_all
+  SQL (15.4ms)  DELETE FROM "users"
+2.1.3 :006 > insert_smart
+   #...snip ~30 lines of output, composing 5 sql queries...
+0.275503
+```
+
+...That's 26.6 seconds for classic create loop... vs. 0.276 seconds for postgres_upsert.  
+This is over 96X faster.  And it only cost me ~6 extra lines of code.
 
-passing :update_only = true will ensure that no new records are created, but records will be updated.
+Note that for the benchmark, my database is local.  The performance improvement should only increase when we have network latency to worry about.
 
 ## Note on Patches/Pull Requests
 

data/Rakefile

@@ -1,18 +1,6 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.expand_path("../lib", __FILE__)
-require 'bundler/gem_tasks'
-require 'rubygems'
-require 'rspec/core/rake_task'
-require 'rdoc/task'
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
 
-task :default => :spec
+require File.expand_path('../config/application', __FILE__)
 
-RSpec::Core::RakeTask.new(:spec)
-
-Rake::RDocTask.new do |rdoc|
-  version = File.exist?('VERSION') ? File.read('VERSION') : ""
-  rdoc.rdoc_dir = 'rdoc'
-  rdoc.title = "postgres_upsert #{version}"
-  rdoc.rdoc_files.include('README*')
-  rdoc.rdoc_files.include('lib/**/*.rb')
-end
+Rails.application.load_tasks