postgres_upsert 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a30c825147ef323c1e0d60d9fc08e9668b6bdf02
-  data.tar.gz: d3f8799b61a273b4abde1eae7a6b38eea3ba614c
+  metadata.gz: 363018cec57166d2976cbeedc60fad9dc17e5ac7
+  data.tar.gz: 280a43ba9f6dea9a72111031c5c7e82089185f72
 SHA512:
-  metadata.gz: 07649b13cae7be995e12c02e5418bde744f932127587141a160bd09f502f9342f2ce5f2038d20953eed9425043f67c056dde96dec78ed5a8fcf5ec0c11392bea
-  data.tar.gz: e963592a3c91c61d206d018cbcc63aa07995a50165c97dbb88e9352e8b16c76cdcbaef85a9c9b655716d0e0ab387c9c155bf1890200374ce074e5405899929fd
+  metadata.gz: 195008d31407158e4cecf27fbd1aec45b6b3c17a078efe5dcd567e4ec8d023f187d597cf0b1a5a1e46e53c901e957a7c70b6ceccdd1aa27231f1cd2f727b194d
+  data.tar.gz: fbf6e4e15fc23ab1c344a1a8bdd9ba80733cc77c776b6eed98c05936a0dd6cd4889f387e43f028c8c380a2e7d4beb9f3ecc58ef5374183ff3891821077e0fb37

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    postgres_upsert (1.0.0)
+    postgres_upsert (1.1.0)
       activerecord (>= 3.0.0)
       pg (~> 0.17.0)
       rails (>= 3.0.0)

data/README.md

@@ -19,9 +19,16 @@ Run the bundle command
 
 The gem will add the aditiontal class method to ActiveRecord::Base
 
-* pg_upsert
+* pg_upsert io_object_or_file_path, [options]
 
-### Using pg_upsert
+io_object_or_file_path => is a file path or an io object (StringIO, FileIO, etc.)
+
+options:
+:delimiter - the string to use to delimit fields.  Default is ","
+:format - the format of the file (valid formats are :csv or :binary).  Default is :csv
+:header => specifies if the file/io source contains a header row.  Either :header option must be true, or :columns list must be passed.  Default true
+:key_column => the primary key or unique key column on your ActiveRecord table, used to distinguish new records from existing records.  Default is the primary_key of your ActiveRecord model class.
+:update_only => when true, postgres_upsert will ONLY update existing records, and not insert new.  Default is false.
 
 pg_upsert will allow you to copy data from an arbritary IO object or from a file in the database server (when you pass the path as string).
 Let's first copy from a file in the database server, assuming again that we have a users table and
@@ -39,17 +46,6 @@ User.pg_upsert "/tmp/users.csv", :map => {'name' => 'first_name'}
 ```
 
 In the above example the header name in the CSV file will be mapped to the field called first_name in the users table.
-You can also manipulate and modify the values of the file being imported before they enter into the database using a block:
-
-```ruby
-User.pg_upsert "/tmp/users.csv" do |row|
-  row[0] = "fixed string"
-end
-```
-
-The above example will always change the value of the first column to "fixed string" before storing it into the database.
-For each iteration of the block row receives an array with the same order as the columns in the CSV file.
-
 
 To copy a binary formatted data file or IO object you can specify the format as binary
 
@@ -85,6 +81,7 @@ User.pg_upsert "/tmp/users.dat", :format => :binary, :key_column => ["external_t
 
 obviously, the field you pass must be a unique key in your database (this is not enforced at the moment, but will be)
 
+passing :update_only = true will ensure that no new records are created, but records will be updated.
 
 ## Note on Patches/Pull Requests
 

data/lib/postgres_upsert/active_record.rb

@@ -7,171 +7,7 @@ module ActiveRecord
     # * You can map fields from the file to different fields in the table using a map in the options hash
     # * For further details on usage take a look at the README.md
     def self.pg_upsert path_or_io, options = {}
-      options.reverse_merge!({:delimiter => ",", :format => :csv, :header => true, :key_column => primary_key})
-      options_string = options[:format] == :binary ? "BINARY" : "DELIMITER '#{options[:delimiter]}' CSV"
-
-      io = path_or_io.instance_of?(String) ? File.open(path_or_io, 'r') : path_or_io
-      columns_list = get_columns(io, options)
-      
-      if columns_list.empty? 
-        raise "Either the :columns option or :header => true are required"
-      end
-      copy_table = get_temp_table_name(options)
-      destination_table = get_table_name(options)
-
-      columns_string = columns_string_for_copy(columns_list)
-      create_temp_table(copy_table, destination_table, columns_list, options) if destination_table
-
-      connection.raw_connection.copy_data %{COPY #{copy_table} #{columns_string} FROM STDIN #{options_string}} do
-        if block_given?
-          block = Proc.new
-        end
-        while line = read_input_line(io, options, &block) do
-          next if line.strip.size == 0
-          connection.raw_connection.put_copy_data line
-        end
-      end
-
-      if destination_table
-        upsert_from_temp_table(copy_table, destination_table, columns_list, options)
-        drop_temp_table(copy_table)
-      end
-    end
-
-    private
-
-    def self.get_columns(io, options)
-      columns_list = options[:columns] || []
-      if options[:format] != :binary && options[:header]
-        #if header is present, we need to strip it from io, whether we use it for the columns list or not.
-        line = io.gets
-          if columns_list.empty?
-            columns_list = line.strip.split(options[:delimiter])
-          end
-      end
-      columns_list = columns_list.map{|c| options[:map][c.to_s] } if options[:map]
-      return columns_list
-    end
-
-    def self.columns_string_for_copy(columns_list)
-      str = get_columns_string(columns_list)
-      str.empty? ? str : "(#{str})"
-    end
-
-    def self.columns_string_for_select(columns_list)
-      columns = columns_list.clone
-      columns << "created_at" if column_names.include?("created_at")
-      columns << "updated_at" if column_names.include?("updated_at")
-      str = get_columns_string(columns)
-    end
-
-    def self.columns_string_for_insert(columns_list)
-      columns = columns_list.clone
-      columns << "created_at" if column_names.include?("created_at")
-      columns << "updated_at" if column_names.include?("updated_at")
-      str = get_columns_string(columns)
-    end
-
-    def self.select_string_for_insert(columns_list)
-      columns = columns_list.clone
-      str = get_columns_string(columns)
-      str << ",'#{DateTime.now.utc}'" if column_names.include?("created_at")
-      str << ",'#{DateTime.now.utc}'" if column_names.include?("updated_at")
-      str
-    end
-
-    def self.select_string_for_create(columns_list, options)
-      columns = columns_list.map(&:to_sym)
-      columns << options[:key_column].to_sym unless columns.include?(options[:key_column].to_sym)
-      get_columns_string(columns)
-    end
-
-    def self.get_columns_string(columns_list)
-      columns_list.size > 0 ? "\"#{columns_list.join('","')}\"" : ""
-    end
-
-    def self.get_table_name(options)
-      if options[:table]
-        connection.quote_table_name(options[:table])
-      else
-        quoted_table_name
-      end
-    end
-
-    def self.get_temp_table_name(options)
-      "#{table_name}_temp_#{rand(1000)}"
-    end
-
-    def self.read_input_line(io, options)
-      if options[:format] == :binary
-        begin
-          return io.readpartial(10240)
-        rescue EOFError
-        end
-      else
-        line = io.gets
-        if block_given? && line
-          row = line.strip.split(options[:delimiter])
-          yield(row)
-          line = row.join(options[:delimiter]) + "\n"
-        end
-        return line
-      end
-    end
-
-    def self.upsert_from_temp_table(temp_table, dest_table, columns_list, options)
-      update_from_temp_table(temp_table, dest_table, columns_list, options)
-      insert_from_temp_table(temp_table, dest_table, columns_list, options)
-    end
-
-    def self.update_from_temp_table(temp_table, dest_table, columns_list, options)
-      ActiveRecord::Base.connection.execute <<-SQL
-        UPDATE #{dest_table} AS d
-          #{update_set_clause(columns_list)}
-          FROM #{temp_table} as t
-          WHERE t.#{options[:key_column]} = d.#{options[:key_column]}
-          AND d.#{options[:key_column]} IS NOT NULL;
-      SQL
-    end
-
-    def self.update_set_clause(columns_list)
-      command = columns_list.map do |col|
-        "\"#{col}\" = t.\"#{col}\""
-      end
-      command << "\"updated_at\" = '#{DateTime.now.utc}'" if column_names.include?("updated_at") 
-      "SET #{command.join(',')}"
-    end
-
-    def self.insert_from_temp_table(temp_table, dest_table, columns_list, options)
-      columns_string = columns_string_for_insert(columns_list)
-      select_string = select_string_for_insert(columns_list)
-      ActiveRecord::Base.connection.execute <<-SQL
-        INSERT INTO #{dest_table} (#{columns_string})
-          SELECT #{select_string}
-          FROM #{temp_table} as t
-          WHERE NOT EXISTS 
-            (SELECT 1 
-                  FROM #{dest_table} as d 
-                  WHERE d.#{options[:key_column]} = t.#{options[:key_column]})
-          AND t.#{options[:key_column]} IS NOT NULL;
-      SQL
-    end
-
-    def self.create_temp_table(temp_table, dest_table, columns_list, options)
-      columns_string = select_string_for_create(columns_list, options)
-      ActiveRecord::Base.connection.execute <<-SQL
-        SET client_min_messages=WARNING;
-        DROP TABLE IF EXISTS #{temp_table};
-
-        CREATE TEMP TABLE #{temp_table} 
-          AS SELECT #{columns_string} FROM #{dest_table} WHERE 0 = 1;
-      SQL
-    end
-
-    def self.drop_temp_table(temp_table)
-      ActiveRecord::Base.connection.execute <<-SQL
-        DROP TABLE #{temp_table} 
-      SQL
+      PostgresUpsert::Writer.new(self, path_or_io, options).write
     end
   end
 end

data/lib/postgres_upsert/writer.rb

@@ -0,0 +1,179 @@
+module PostgresUpsert
+
+  class Writer
+
+    def initialize(klass, source, options = {})
+      @klass = klass
+      @options = options.reverse_merge({
+        :delimiter => ",", 
+        :format => :csv, 
+        :header => true, 
+        :key_column => @klass.primary_key,
+        :update_only => false})
+      @source = source.instance_of?(String) ? File.open(source, 'r') : source
+      @columns_list = get_columns
+      generate_temp_table_name
+    end
+
+    def write
+      if @columns_list.empty? 
+        raise "Either the :columns option or :header => true are required"
+      end
+
+      csv_options = @options[:format] == :binary ? "BINARY" : "DELIMITER '#{@options[:delimiter]}' CSV"
+
+      copy_table = @temp_table_name
+      destination_table = get_table_name
+
+      columns_string = columns_string_for_copy
+      create_temp_table
+
+      ActiveRecord::Base.connection.raw_connection.copy_data %{COPY #{copy_table} #{columns_string} FROM STDIN #{csv_options}} do
+
+        while line = read_input_line do
+          next if line.strip.size == 0
+          ActiveRecord::Base.connection.raw_connection.put_copy_data line
+        end
+      end
+
+      if destination_table
+        upsert_from_temp_table
+        drop_temp_table
+      end
+    end
+
+  private
+
+    def get_columns
+      columns_list = @options[:columns] || []
+      if @options[:format] != :binary && @options[:header]
+        #if header is present, we need to strip it from io, whether we use it for the columns list or not.
+        line = @source.gets
+          if columns_list.empty?
+            columns_list = line.strip.split(@options[:delimiter])
+          end
+      end
+      columns_list = columns_list.map{|c| @options[:map][c.to_s] } if @options[:map]
+      return columns_list
+    end
+
+    def columns_string_for_copy
+      str = get_columns_string
+      str.empty? ? str : "(#{str})"
+    end
+
+    def columns_string_for_select
+      columns = @columns_list.clone
+      columns << "created_at" if @klass.column_names.include?("created_at")
+      columns << "updated_at" if @klass.column_names.include?("updated_at")
+      str = get_columns_string(columns)
+    end
+
+    def columns_string_for_insert
+      columns = @columns_list.clone
+      columns << "created_at" if @klass.column_names.include?("created_at")
+      columns << "updated_at" if @klass.column_names.include?("updated_at")
+      str = get_columns_string(columns)
+    end
+
+    def select_string_for_insert
+      columns = @columns_list.clone
+      str = get_columns_string(columns)
+      str << ",'#{DateTime.now.utc}'" if @klass.column_names.include?("created_at")
+      str << ",'#{DateTime.now.utc}'" if @klass.column_names.include?("updated_at")
+      str
+    end
+
+    def select_string_for_create
+      columns = @columns_list.map(&:to_sym)
+      columns << @options[:key_column].to_sym unless columns.include?(@options[:key_column].to_sym)
+      get_columns_string(columns)
+    end
+
+    def get_columns_string(columns = nil)
+      columns ||= @columns_list
+      columns.size > 0 ? "\"#{columns.join('","')}\"" : ""
+    end
+
+    def get_table_name
+      if @options[:table]
+        connection.quote_table_name(@options[:table])
+      else
+        @klass.quoted_table_name
+      end
+    end
+
+    def generate_temp_table_name
+      @temp_table_name = "#{@klass.table_name}_temp_#{rand(1000)}"
+    end
+
+    def read_input_line
+      if @options[:format] == :binary
+        begin
+          return @source.readpartial(10240)
+        rescue EOFError
+        end
+      else
+        line = @source.gets
+        return line
+      end
+    end
+
+    def upsert_from_temp_table
+      update_from_temp_table
+      insert_from_temp_table unless @options[:update_only]
+    end
+
+    def update_from_temp_table
+      ActiveRecord::Base.connection.execute <<-SQL
+        UPDATE #{get_table_name} AS d
+          #{update_set_clause}
+          FROM #{@temp_table_name} as t
+          WHERE t.#{@options[:key_column]} = d.#{@options[:key_column]}
+          AND d.#{@options[:key_column]} IS NOT NULL;
+      SQL
+    end
+
+    def update_set_clause
+      command = @columns_list.map do |col|
+        "\"#{col}\" = t.\"#{col}\""
+      end
+      command << "\"updated_at\" = '#{DateTime.now.utc}'" if @klass.column_names.include?("updated_at") 
+      "SET #{command.join(',')}"
+    end
+
+    def insert_from_temp_table
+      columns_string = columns_string_for_insert
+      select_string = select_string_for_insert
+      ActiveRecord::Base.connection.execute <<-SQL
+        INSERT INTO #{get_table_name} (#{columns_string})
+          SELECT #{select_string}
+          FROM #{@temp_table_name} as t
+          WHERE NOT EXISTS 
+            (SELECT 1 
+                  FROM #{get_table_name} as d 
+                  WHERE d.#{@options[:key_column]} = t.#{@options[:key_column]})
+          AND t.#{@options[:key_column]} IS NOT NULL;
+      SQL
+    end
+
+    def create_temp_table
+      columns_string = select_string_for_create
+      ActiveRecord::Base.connection.execute <<-SQL
+        SET client_min_messages=WARNING;
+        DROP TABLE IF EXISTS #{@temp_table_name};
+
+        CREATE TEMP TABLE #{@temp_table_name} 
+          AS SELECT #{columns_string} FROM #{get_table_name} WHERE 0 = 1;
+      SQL
+    end
+
+    def drop_temp_table
+      ActiveRecord::Base.connection.execute <<-SQL
+        DROP TABLE #{@temp_table_name} 
+      SQL
+    end
+  end
+
+
+end

data/lib/postgres_upsert.rb

@@ -1,6 +1,7 @@
 require 'rubygems'
 require 'active_record'
 require 'postgres_upsert/active_record'
+require 'postgres_upsert/writer'
 require 'rails'
 
 class PostgresCopy < Rails::Railtie

data/postgres_upsert.gemspec

@@ -5,7 +5,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name = "postgres_upsert"
-  s.version = "1.1.0"
+  s.version = "2.0.0"
 
   s.platform    = Gem::Platform::RUBY
   s.required_ruby_version     = ">= 1.8.7"

data/spec/pg_upsert_csv_spec.rb

@@ -64,29 +64,6 @@ describe "pg_upsert from file with CSV format" do
     ).to include('data' => 'test data 1', 'created_at' => timestamp, 'updated_at' => timestamp)
   end
 
-  it "should import and allow changes in block" do
-    TestModel.pg_upsert(File.open(File.expand_path('spec/fixtures/comma_with_header.csv'), 'r')) do |row|
-      row[1] = 'changed this data'
-    end
-    expect(
-      TestModel.first.attributes
-    ).to include('data' => 'changed this data', 'created_at' => timestamp, 'updated_at' => timestamp)
-  end
-
-  it "should import 2 lines and allow changes in block" do
-    TestModel.pg_upsert(File.open(File.expand_path('spec/fixtures/tab_with_two_lines.csv'), 'r'), :delimiter => "\t") do |row|
-      row[1] = 'changed this data'
-    end
-
-    expect(
-      TestModel.find(1).attributes
-    ).to include('data' => 'changed this data', 'created_at' => timestamp, 'updated_at' => timestamp)
-    expect(
-      TestModel.find(2).attributes
-    ).to include('data' => 'changed this data', 'created_at' => timestamp, 'updated_at' => timestamp)
-    expect(TestModel.count).to eq 2
-  end
-
   it "should not expect a header when :header is false" do
     TestModel.pg_upsert(File.open(File.expand_path('spec/fixtures/comma_without_header.csv'), 'r'), :header => false, :columns => [:id,:data])
 
@@ -206,5 +183,24 @@ describe "pg_upsert from file with CSV format" do
       ).to include("id" => 1, "data" => "old stuff", "extra" => "ABC: Always Be Changing.")
     end
   end
+
+  context 'update only' do
+    let(:original_created_at) {5.days.ago.utc}
+    before(:each) do
+      TestModel.create(id: 1, data: "From the before time, in the long long ago", :created_at => original_created_at)
+    end
+    it 'will only update and not insert if insert_only flag is passed.' do
+      TestModel.pg_upsert File.expand_path('spec/fixtures/tab_with_two_lines.csv'), :delimiter => "\t", :update_only => true
+
+      expect(
+        TestModel.find(1).attributes
+      ).to eq("id"=>1, "data"=>"test data 1", "created_at" => original_created_at  , "updated_at" => timestamp)
+      expect{
+        TestModel.find(2)
+      }.to raise_error(ActiveRecord::RecordNotFound)
+
+    end
+
+  end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: postgres_upsert
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Steve Mitchell
@@ -124,6 +124,7 @@ files:
 - VERSION
 - lib/postgres_upsert.rb
 - lib/postgres_upsert/active_record.rb
+- lib/postgres_upsert/writer.rb
 - postgres_upsert.gemspec
 - spec/fixtures/2_col_binary_data.dat
 - spec/fixtures/comma_with_header.csv