sequel-pg_bulk_upsert 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dae595e4106d5d417ee04cc133ec517bb262a258
+  data.tar.gz: 8487d7990c2ddc1b532d9ccf73f6b5a6d49d9635
+SHA512:
+  metadata.gz: 0b5a2f42be945d76e6dc3a0ba36c0674203c9b4415e6b1860d8d9f6c075ec311bd7afbbce1bd5ba7787d4c2fb45f7c68c674483b29e0f88489ad43e2aa676bf8
+  data.tar.gz: 489f39454ff4bfe11acad6816454d0e693821b9454d912a2a9930e6b61597bed98850d12256bab88deec651991fe9410c1a1d56daa2f7c585480f6179e9026ee

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in sequel-pg-bulk-upsert.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Juan Manuel Barreneche
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+# Sequel::PgBulkUpsert
+
+This Sequel extension implements `on_duplicate_key_update(*...).multi_insert(*...)` for postgresql. The
+syntax is 100% with the one that Sequel provides for mysql with a couple of caveats.
+
+There are different ways for implementing "upsert" in postgresql, some of them discussed in:
+
+http://johtopg.blogspot.com.ar/2014/04/upsertisms-in-postgres.html
+
+Because we aim to do bulk upserts, this extension uses the one discribed here:
+
+http://tapoueh.org/blog/2013/03/15-batch-update
+
+It consists in 3 parts:
+
+1. Create a temp table with the target table structure
+
+2. Fill the temp table with the rows that are going to be "upserted"
+
+3. Updates the records that exist in the target table with the temp table information, then, by doing an
+   "anti-join" between the temp table and the target table, the records that didn't exist are inserted.
+
+   The resulting SQL looks like this:
+
+   ```SQL
+      WITH "update_cte" AS
+        (UPDATE "<target_table>"
+            SET "updatable_column" = "<temp_table>"."updatable_column"
+           FROM "<temp_table>"
+           WHERE ("<target_table>"."id" = "<temp_table>"."id")
+           RETURNING "<target_table>"."id")
+      INSERT INTO "<target_table>" ("updatable_column", "insertable_column")
+       SELECT "updatable_column", "insertable_column"
+       FROM "<temp_table>" LEFT JOIN "update_cte" USING ("id")
+       WHERE ("update_cte"."id" IS NULL)
+       RETURNING "<target_table>"."id"
+   ```
+
+## Caveats
+
+1. The target table **must** have a primary key (this is the key used to decide when a record "exist"), it
+   won't work for any other unique constraint in the target table.
+
+2. This strategy for upsert isn't suitable for concurrent upserts on the same table. Because the concurrent
+  inserts will see the same table, and neither will do an update on the others records, which will result in
+  a duplicate key error.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'sequel-pg-bulk-upsert'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sequel-pg-bulk-upsert
+
+## Usage
+
+```ruby
+require 'sequel/pg_bulk_upsert'
+
+# This can also be done for a specify dataset
+DB.extension(:pg_bulk_upsert)
+
+DB[:target].on_duplicate_key_update(:column1, :column2).multi_insert([
+  {column1: '1', column2: '2', column3: '3'},
+  {column1: '4', column2: '4', column3: '4'}
+])
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new :test do |t|
+  t.libs << "test"
+  # t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end
+
+task default: %w[test]

data/lib/sequel-pg_bulk_upsert.rb

@@ -0,0 +1 @@
+require 'sequel-pg_bulk_upsert/version'

data/lib/sequel-pg_bulk_upsert/version.rb

@@ -0,0 +1,5 @@
+module Sequel
+  module PgBulkUpsert
+    VERSION = "0.1.0"
+  end
+end

data/lib/sequel/extensions/pg_bulk_upsert.rb

@@ -0,0 +1,77 @@
+require "sequel"
+
+module Sequel
+  Dataset::NON_SQL_OPTIONS << :on_duplicate_key_update
+
+  module PgBulkUpsert
+    def on_duplicate_key_update(*args)
+      clone(on_duplicate_key_update: args)
+    end
+
+    def multi_insert_sql(columns, values)
+      if duplicate_keys = @opts[:on_duplicate_key_update]
+        from_table_name = @opts[:from].first # XXX How: can we ensure theres only one?
+        temp_table_name = "#{from_table_name}_tmp_#{Time.now.strftime("%s%L")}"
+        upsert_on = @db.primary_key(from_table_name) or raise "missing primary_key for #{from_table_name}"
+
+        [
+          create_temp_table_from_existing_sql(from_table_name, temp_table_name),
+          multi_insert_without_duplicates_sqls(temp_table_name, columns, values),
+          upsert_from_to_sql(temp_table_name, from_table_name, upsert_on.to_sym, duplicate_keys, columns)
+        ].flatten
+      else
+        super
+      end
+    end
+
+    private
+
+    def multi_insert_without_duplicates_sqls(target_table, columns, values)
+      clone(on_duplicate_key_update: nil).
+        from(target_table).
+        multi_insert_sql(columns, values)
+    end
+
+    def upsert_from_to_sql(source_name, target_name, join_on, update_columns, insert_columns)
+      target = @db.from(target_name)
+      source = @db.from(source_name)
+      source_ds = source.
+        select(*insert_columns).
+        left_join(:update_cte, [join_on]).
+        where(Sequel.qualify(:update_cte, join_on) => nil)
+
+      update_hash = update_columns.each_with_object({}) do |column, hash|
+        hash[column] = Sequel.qualify(source_name, column)
+      end
+
+      target.with(:update_cte,
+        target.from(target_name, source_name).
+          where(Sequel.qualify(target_name, join_on) => Sequel.qualify(source_name, join_on)).
+          returning(Sequel.qualify(target_name, join_on)).with_sql(:update_sql, update_hash)).
+        returning(Sequel.qualify(target_name, join_on)).insert_sql(insert_columns, source_ds)
+    end
+
+    def create_temp_table_from_existing_sql(base_table, temp_table_name)
+      columns_information = @db.schema(base_table)
+
+      generator = @db.create_table_generator do
+        columns_information.each do |col, data|
+          column col, data[:db_type], primary_key: data[:primary]
+        end
+      end
+
+      @db.send :create_table_sql, temp_table_name, generator, temp: true, on_commit: :drop
+    end
+
+  end
+
+  module PgDatabaseBulkUpsert
+    def self.extended(db)
+      db.extend_datasets(PgBulkUpsert)
+    end
+  end
+
+  Dataset.register_extension :pg_bulk_upsert, PgBulkUpsert
+  Database.register_extension :pg_bulk_upsert, PgDatabaseBulkUpsert
+end
+

data/sequel-pg_bulk_upsert.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sequel-pg_bulk_upsert/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "sequel-pg_bulk_upsert"
+  gem.version       = Sequel::PgBulkUpsert::VERSION
+  gem.authors       = ["Juan Manuel Barreneche"]
+  gem.email         = ["jbarreneche@restorando.com"]
+  gem.description   = %q{Implementation `on_duplicate_key_update(*...).multi_insert(*...)` for postgresql}
+  gem.summary       = %q{Implementation `on_duplicate_key_update(*...).multi_insert(*...)` for postgresql}
+  gem.homepage      = "https://github.com/restorando/sequel-pg_bulk_upsert"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.add_dependency "sequel", ">= 4.15.0"
+  gem.add_dependency "pg"
+  gem.add_development_dependency "minitest"
+  gem.add_development_dependency "pry"
+  gem.require_paths = ["lib"]
+end

data/test/lib/dataset_extension_test.rb

@@ -0,0 +1,95 @@
+require 'test_helper'
+
+class DatasetExtensionTest < MiniTest::Test
+
+  def setup
+    @db = Sequel.
+      connect('mock://postgres').
+      extension(:pg_bulk_upsert)
+
+    def @db.schema(table)
+      if table == :target
+        [
+          [:id, { primary: true, db_type: :serial }],
+          [:updatable_column,  { db_type: :text   }],
+          [:insertable_column, { db_type: :text   }]
+        ]
+      end
+    end
+
+    @db.create_table(:target) do
+      primary_key :id
+      String :updatable_column
+      String :insertable_column
+    end
+
+    @ds = @db.dataset.from(:target)
+
+    @upsert_columns = [:updatable_column, :insertable_column]
+    @upsert_data = [%w[foo bar]]
+
+    # Clear setup sqls
+    @db.sqls
+  end
+
+  def test_temp_table_creation
+    sqls = do_upsert
+    temp_table_name   = extract_temp_table_name(sqls[1])
+
+    temp_table_creation = sqls[1]
+    assert_equal strip_heredoc(<<-SQL).gsub("\n", ""), temp_table_creation
+      CREATE TEMPORARY TABLE "#{temp_table_name}"
+       ("id" serial PRIMARY KEY, "updatable_column" text, "insertable_column" text) ON COMMIT DROP
+    SQL
+  end
+
+  def test_temp_table_batch_loading
+    sqls = do_upsert
+    temp_table_name   = extract_temp_table_name(sqls[1])
+    temp_table_insert = @db.from(temp_table_name).multi_insert_sql(@upsert_columns, @upsert_data)
+
+    assert_equal temp_table_insert, [sqls[2]]
+  end
+
+  def test_upsert_from_temp
+    sqls = do_upsert
+    temp_table_name   = extract_temp_table_name(sqls[1])
+    temp_table_insert = strip_heredoc(<<-SQL).gsub("\n", "")
+      WITH "update_cte" AS
+       (UPDATE "target" SET "updatable_column" = "#{temp_table_name}"."updatable_column"
+       FROM "#{temp_table_name}" WHERE ("target"."id" = "#{temp_table_name}"."id") RETURNING "target"."id")
+       INSERT INTO "target" ("updatable_column", "insertable_column")
+       SELECT "updatable_column", "insertable_column"
+       FROM "#{temp_table_name}" LEFT JOIN "update_cte" USING ("id")
+       WHERE ("update_cte"."id" IS NULL)
+       RETURNING "target"."id"
+    SQL
+
+    assert_equal temp_table_insert, sqls[3]
+  end
+
+  def test_upsert_inside_transaction
+    sqls = do_upsert
+
+    assert_equal "BEGIN",  sqls[0]
+    assert_equal "COMMIT", sqls[4]
+  end
+
+  private
+
+  def do_upsert(columns = @upsert_columns, data = @upsert_data)
+    @ds.on_duplicate_key_update(:updatable_column).import(columns, data)
+
+    @db.sqls
+  end
+
+  def extract_temp_table_name(create_sql)
+    create_sql[/CREATE TEMPORARY TABLE "(\w+)"/i, 1]
+  end
+
+  def strip_heredoc(heredoc)
+    indent = (heredoc.scan(/^[ \t]*(?=\S)/).min || "").size
+    heredoc.gsub(/^[ \t]{#{indent}}/, '')
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'bundler/setup'
+require 'minitest/autorun'
+
+require 'sequel'
+require 'sequel-pg_bulk_upsert'

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: sequel-pg_bulk_upsert
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Juan Manuel Barreneche
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.15.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Implementation `on_duplicate_key_update(*...).multi_insert(*...)` for
+  postgresql
+email:
+- jbarreneche@restorando.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sequel-pg_bulk_upsert.rb
+- lib/sequel-pg_bulk_upsert/version.rb
+- lib/sequel/extensions/pg_bulk_upsert.rb
+- sequel-pg_bulk_upsert.gemspec
+- test/lib/dataset_extension_test.rb
+- test/test_helper.rb
+homepage: https://github.com/restorando/sequel-pg_bulk_upsert
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Implementation `on_duplicate_key_update(*...).multi_insert(*...)` for postgresql
+test_files:
+- test/lib/dataset_extension_test.rb
+- test/test_helper.rb