em-pg-client-helper 1.3.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f507ec3c80d6ed2c58575898e2132bfa3e05ac36
-  data.tar.gz: 5c311e9a44d3274689c4c6f12dfc028a3e721328
+  metadata.gz: 54910da339d3b8be844ddddb6b373dbec7ca04b5
+  data.tar.gz: efb8474eb8d13f6020116b4923dfea5f4793059c
 SHA512:
-  metadata.gz: 6b39ee8f0cdce4de503b1793ca26f13b0bd493cd53b80390a95a65018252af5d30c837f733013fb07678a8aab32caa62fceab8d0f6a26b8c70494df3185c2173
-  data.tar.gz: 0af1627ff7f3bac097818c5a91f5c71467f32aadfb18fef17dd939db9e360e1a20a8559a68421b5ae24f0d74153907ca9af62c3d3d5347ebf18fda5662239cf0
+  metadata.gz: c39f44d77f9238e333d22fd9ed3a6970f46c45a783d3414f733baa61a912479524032c962fdacd0e085e56f38c0615d3dba3e30fcbe66198168ab657a1c113ba
+  data.tar.gz: 012405e9483038a357080c994b20ce3ce0793379e557b28f04331a34d70a07a2871afeadc98f1c6303d5d3397073fb61db0ff7b4a4c8826f8fff2cb7aee82055

data/lib/em-pg-client-helper.rb

@@ -76,10 +76,10 @@ module PG::EM::Client::Helper
 		sqls = sqldb.sqls
 
 		if sqls.empty?
-			sqls = [ret.sql] rescue []
+			sqls = [ret.sql] rescue ret
 		end
 
-		if sqls.empty?
+		if sqls.nil? or sqls.empty?
 			raise PG::EM::Client::Helper::BadSequelError,
 			      "Your block did not generate an SQL statement"
 		end
@@ -195,6 +195,58 @@ module PG::EM::Client::Helper
 		db.exec_defer(*insert_sql(tbl, params))
 	end
 
+	# Efficiently perform a "bulk" insert of multiple rows.
+	#
+	# When you have a large quantity of data to insert into a table, you don't
+	# want to do it one row at a time -- that's *really* inefficient.  On the
+	# other hand, if you do one giant multi-row insert statement, the insert
+	# will fail if *any* of the rows causes a constraint failure.  What to do?
+	#
+	# Well, here's our answer: try to insert all the records at once.  If that
+	# fails with a constraint violation, then split the set of records in half
+	# and try to bulk insert each of those halves.  Recurse in this fashion until
+	# you only have one record to insert.
+	#
+	# @param db [PG::EM::Client, PG::EM::ConnectionPool] the connection against
+	#   which the insert wil be run.
+	#
+	# @param tbl [#to_sym] the name of the table into which you wish to insert
+	#   your data.
+	#
+	# @param columns [Array<#to_sym>] the columns into which each record of data
+	#   will be inserted.
+	#
+	# @param rows [Array<Array<Object>>] the values to insert.  Each entry in
+	#   the outermost array is a row of data; the elements of each of these inner
+	#   arrays corresponds to the column in the same position in the `columns`
+	#   array.  **NOTE**: we don't do any checking to make sure you're giving
+	#   us the correct list of values for each row.  Thus, if you give us a
+	#   row array that has too few, or too many, entries, the database will puke.
+	#
+	# @return [EM::Deferrable] the deferrable in which the query is being called;
+	#   once the bulk insert completes successfully, the deferrable will succeed
+	#   with the number of rows that were successfully inserted.  If the insert
+	#   could not be completed, the deferrable will fail (`#errback`) with the
+	#   exception.
+	#
+	# @since 2.0.0
+	#
+	def db_bulk_insert(db, tbl, columns, rows, &blk)
+		EM::Completion.new.tap do |df|
+			df.callback(&blk) if blk
+
+			db_transaction(db) do |txn|
+				txn.bulk_insert(tbl, columns, rows) do |count|
+					txn.commit do
+						df.succeed(count)
+					end
+				end
+			end.errback do |ex|
+				df.fail(ex)
+			end
+		end
+	end
+
 	# @!macro upsert_params
 	#
 	#   @param tbl [#to_s] The name of the table on which to operate.

data/lib/em-pg-client-helper/deferrable_group.rb

@@ -7,6 +7,13 @@
 class PG::EM::Client::Helper::DeferrableGroup
 	include ::EventMachine::Deferrable
 
+	# Raised when an attempt is made to add a deferrable to a group on
+	# which {#close} has already been called.
+	#
+	# @since 2.0.0
+	#
+	class ClosedError < StandardError; end
+
 	# Create a new deferrable group.
 	#
 	def initialize
@@ -23,15 +30,15 @@ class PG::EM::Client::Helper::DeferrableGroup
 	#
 	# @return [EM::Deferrable] the same deferrable.
 	#
-	# @raise [RuntimeError] if you attempt to add a deferrable after the
-	#   group has been closed (that is, the `#close` method has been called),
-	#   indicating that the deferrable group doesn't have any more deferrables
-	#   to add.
+	# @raise [PG::EM::Client::Helper::DeferrableGroup::ClosedError] if you
+	#   attempt to add a deferrable after the group has been closed (that is,
+	#   the `#close` method has been called), indicating that the deferrable
+	#   group doesn't have any more deferrables to add.
 	#
 	def add(df)
 		if @closed
-			raise RuntimeError,
-			      "This deferrable group is closed."
+			raise ClosedError,
+			      "This deferrable group is closed"
 		end
 
 		@outstanding << df

data/lib/em-pg-client-helper/transaction.rb

@@ -1,3 +1,5 @@
+require 'securerandom'
+
 # Represents a database transaction, and contains all of the methods which
 # can be used to execute queries within the transaction connection.
 #
@@ -5,6 +7,13 @@ class PG::EM::Client::Helper::Transaction
 	include ::PG::EM::Client::Helper
 	include ::EventMachine::Deferrable
 
+	# Raised when you attempt to execute a query in a transaction which has
+	# been finished (either by COMMIT or ROLLBACK).
+	#
+	# @since 2.0.0
+	#
+	class ClosedError < StandardError; end
+
 	# Create a new transaction.  You shouldn't have to call this yourself;
 	# `db_transaction` should create one and pass it to your block.
 	#
@@ -21,12 +30,12 @@ class PG::EM::Client::Helper::Transaction
 	# @raise [ArgumentError] If an unknown isolation level is specified.
 	#
 	def initialize(conn, opts = {}, &blk)
-		@conn       = conn
-		@opts       = opts
-		# This can be `nil` if the txn is in progress, or it will be
-		# true or false to indicate success/failure of the txn
-		@committed  = nil
-		@retryable  = opts[:retry]
+		@conn                  = conn
+		@opts                  = opts
+		@finished              = nil
+		@retryable             = opts[:retry]
+		@autorollback_on_error = true
+		@savepoint_stack       = []
 
 		DeferrableGroup.new do |dg|
 			@dg = dg
@@ -59,7 +68,7 @@ class PG::EM::Client::Helper::Transaction
 				end
 			end
 		end.callback do
-			rollback(RuntimeError.new("txn.commit was not called")) unless @committed
+			rollback(RuntimeError.new("txn.commit was not called")) unless @finished
 			self.succeed
 		end.errback do |ex|
 			if @retryable and [PG::TRSerializationFailure].include?(ex.class)
@@ -81,16 +90,16 @@ class PG::EM::Client::Helper::Transaction
 	# exception will be raised.
 	#
 	def commit
-		if @committed.nil?
+		unless @finished
 			trace_query("COMMIT")
 			@conn.exec_defer("COMMIT", []).tap do |df|
 				@dg.add(df)
 			end.callback do
-				@committed = true
+				@finished = true
 				@dg.close
-			end.errback do |ex|
-				@committed = false
-				@dg.fail(ex)
+				yield if block_given?
+			end.errback do
+				@finished = true
 				@dg.close
 			end
 		end
@@ -102,15 +111,81 @@ class PG::EM::Client::Helper::Transaction
 	# event of a database error or other exception.
 	#
 	def rollback(ex)
-		if @committed.nil?
-			exec("ROLLBACK") do
-				@committed = false
-				@dg.fail(ex)
+		unless @finished
+			if @savepoint_stack.empty?
+				exec("ROLLBACK") do
+					@finished = true
+					@dg.fail(ex)
+					@dg.close
+				end
+			else
+				sp = @savepoint_stack.pop
+				exec("ROLLBACK TO \"#{sp[:savepoint]}\"")
+				sp[:deferrable].fail(ex)
 				@dg.close
+				@dg = sp[:parent_deferrable_group]
 			end
 		end
 	end
 
+	# Manage the "rollback on the failure of a single query" behaviour.
+	#
+	# The default behaviour of a transaction, when a query fails, is for
+	# the transaction to automatically be rolled back and the rest of the
+	# statements to not be executed.  In **ALMOST** every case, this is the
+	# correct behaviour.  However, there are some corner cases in which you
+	# want to be able to avoid this behaviour, and will manually react to
+	# the transaction failure in some way.  In that case, you can set this
+	# to `false` and the transaction will not automatically fail.
+	#
+	# Given that pretty much the only thing you can do when a query fails,
+	# other than abort the transaction, is to rollback to a savepoint, you
+	# might want to look at {#savepoint} before you try using this.
+	#
+	# @since 2.0.0
+	#
+	attr_accessor :autorollback_on_error
+
+	# Setup a "savepoint" within the transaction.
+	#
+	# A savepoint is, as the name suggests, kinda like a "saved game", in an
+	# SQL transaction.  If a query fails within a transaction, normally all
+	# you can do is rollback and abort the entire transaction.  Savepoints
+	# give you another option: roll back to the savepoint, and try again.
+	#
+	# So, that's what this method does.  Inside of the block passed to
+	# `#savepoint`, if any query fails, instead of rolling back the entire
+	# transaction, we instead only rollback to the savepoint, and execution
+	# continues by executing the `errback` callbacks defined on the savepoint
+	# deferrable.
+	#
+	# @return [EM::Deferrable]
+	#
+	# @since 2.0.0
+	#
+	def savepoint(&blk)
+		savepoint = SecureRandom.uuid
+		parent_dg = @dg
+		DeferrableGroup.new do |dg|
+			@dg = dg
+
+			dg.callback do
+				@dg = parent_dg
+				@dg.close
+			end
+
+			exec("SAVEPOINT \"#{savepoint}\"").tap do |df|
+				@savepoint_stack << { :savepoint => savepoint,
+				                      :deferrable => df,
+				                      :parent_deferrable_group => parent_dg
+				                    }
+
+				df.callback(&blk) if blk
+			end
+		end
+	end
+
+
 	# Generate SQL statements via Sequel, and run the result against the
 	# database.  Very chic.
 	#
@@ -131,6 +206,67 @@ class PG::EM::Client::Helper::Transaction
 		exec(*insert_sql(tbl, params), &blk)
 	end
 
+	# Efficiently perform a "bulk" insert of multiple rows.
+	#
+	# When you have a large quantity of data to insert into a table, you don't
+	# want to do it one row at a time -- that's *really* inefficient.  On the
+	# other hand, if you do one giant multi-row insert statement, the insert
+	# will fail if *any* of the rows causes a constraint failure.  What to do?
+	#
+	# Well, here's our answer: try to insert all the records at once.  If that
+	# fails with a constraint violation, then split the set of records in half
+	# and try to bulk insert each of those halves.  Recurse in this fashion until
+	# you only have one record to insert.
+	#
+	# @param tbl [#to_sym] the name of the table into which you wish to insert
+	#   your data.
+	#
+	# @param columns [Array<#to_sym>] the columns into which each record of data
+	#   will be inserted.
+	#
+	# @param rows [Array<Array<Object>>] the values to insert.  Each entry in
+	#   the outermost array is a row of data; the elements of each of these inner
+	#   arrays corresponds to the column in the same position in the `columns`
+	#   array.  **NOTE**: we don't do any checking to make sure you're giving
+	#   us the correct list of values for each row.  Thus, if you give us a
+	#   row array that has too few, or too many, entries, the database will puke.
+	#
+	# @yield [Integer] Once the insert has completed, the number of rows that
+	#   were successfully inserted (that may be less than `rows.length` if
+	#   there were any constraint failures) will be yielded to the block.
+	#
+	# @since 2.0.0
+	#
+	def bulk_insert(tbl, columns, rows, &blk)
+		db = Sequel.connect("mock://postgres")
+
+		# Guh hand-hacked SQL is fugly... but what I'm doing is so utterly
+		# niche that Sequel doesn't support it.
+		q_tbl = db.literal(tbl.to_sym)
+		q_cols = columns.map { |c| db.literal(c.to_sym) }
+
+		subselect = "SELECT 1 FROM #{q_tbl} AS dst WHERE " +
+		            q_cols.map { |c| "src.#{c}=dst.#{c}" }.join(" AND ")
+
+		total_rows_inserted = 0
+		DeferrableGroup.new.tap do |dg|
+			rows.each_slice(100) do |slice|
+				vals = slice.map do |row|
+				         "(" + row.map { |v| db.literal(v) }.join(", ") + ")"
+				       end.join(", ")
+				q = "INSERT INTO #{q_tbl} (SELECT * FROM (VALUES #{vals}) " +
+				    "AS src (#{q_cols.join(", ")}) WHERE NOT EXISTS (#{subselect}))"
+				df = exec(q) do |res|
+					total_rows_inserted += res.cmd_tuples
+				end
+				dg.add(df)
+			end
+			dg.callback { dg.succeed(total_rows_inserted) }
+			dg.callback(&blk) if blk
+			dg.close
+		end
+	end
+
 	# Run an upsert inside a transaction.
 	#
 	# @see {PG::EM::Client::Helper#upsert_sql} for all the parameters.
@@ -157,8 +293,8 @@ class PG::EM::Client::Helper::Transaction
 	#   specific query finishes.
 	#
 	def exec(sql, values=[], &blk)
-		unless @committed.nil?
-			raise RuntimeError,
+		if @finished
+			raise ClosedError,
 			      "Cannot execute a query in a transaction that has been closed"
 		end
 
@@ -167,11 +303,13 @@ class PG::EM::Client::Helper::Transaction
 			@dg.add(df)
 			df.callback(&blk) if blk
 		end.errback do |ex|
-			rollback(ex)
+			rollback(ex) if @autorollback_on_error
 		end
 	end
 	alias_method :exec_defer, :exec
 
+	private
+
 	# Trace queries as they happen, if `ENV['EM_PG_TXN_TRACE']` is set.
 	#
 	def trace_query(q, v=nil)

data/spec/db_bulk_insert_spec.rb

@@ -0,0 +1,27 @@
+require_relative './spec_helper'
+
+describe "PG::EM::Client::Helper#db_bulk_insert" do
+	let(:mock_conn) { double(PG::EM::Client) }
+
+	it "inserts multiple records" do
+		expect(dbl = double).to receive(:results).with(2)
+
+		in_em do
+			expect_query("BEGIN")
+		   expect_query('INSERT INTO "foo" ' +
+		                '(SELECT * FROM (VALUES (1, \'x\'), (3, \'y\')) ' +
+		                'AS src ("bar", "baz") ' +
+		                'WHERE NOT EXISTS ' +
+		                '(SELECT 1 FROM "foo" AS dst ' +
+		                'WHERE src."bar"=dst."bar" AND src."baz"=dst."baz"))',
+		                [], 0.001, :succeed, Struct.new(:cmd_tuples).new(2)
+		               )
+		   expect_query("COMMIT")
+
+			db_bulk_insert(mock_conn, "foo", [:bar, :baz], [[1, "x"], [3, "y"]]) do |count|
+				dbl.results(count)
+				EM.stop
+			end
+		end
+	end
+end

data/spec/db_transaction_savepoint_spec.rb

@@ -0,0 +1,118 @@
+require_relative './spec_helper'
+
+describe "PG::EM::Client::Helper::Transaction#savepoint" do
+	let(:mock_conn) { double(PG::EM::Client) }
+
+	it "executes through the savepoint in normal operation" do
+		in_em do
+			expect(SecureRandom).to receive(:uuid).and_return("faff")
+			expect_query("BEGIN")
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wombat"])
+			expect_query('SAVEPOINT "faff"')
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["baz"])
+			expect_query("COMMIT")
+			in_transaction do |txn|
+				txn.insert("foo", :bar => 'wombat') do
+					txn.savepoint do
+						txn.insert("foo", :bar => 'baz') do
+							txn.commit
+						end
+					end.errback do
+						txn.rollback  # Just to show this *doesn't* happen
+					end
+				end
+			end
+		end
+	end
+
+	it "rolls back to the savepoint after a failed INSERT" do
+		in_em do
+			expect(SecureRandom).to receive(:uuid).and_return("faff")
+			expect_query("BEGIN")
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wombat"])
+			expect_query('SAVEPOINT "faff"')
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["baz"])
+			expect_query('ROLLBACK TO "faff"')
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wibble"])
+			expect_query("COMMIT")
+			in_transaction do |txn|
+				txn.insert("foo", :bar => 'wombat') do
+					txn.savepoint do
+						txn.insert("foo", :bar => 'baz') do
+							txn.rollback  # Just to show this *doesn't* happen
+						end
+					end.errback do
+						txn.insert("foo", :bar => 'wibble') do
+							txn.commit
+						end
+					end
+				end
+			end
+		end
+	end
+
+	it "issues ROLLBACK if query fails outside of savepoint" do
+		in_em do
+			expect(SecureRandom).to receive(:uuid).and_return("faff")
+			expect_query("BEGIN")
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wombat"])
+			expect_query('SAVEPOINT "faff"')
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["baz"])
+			# I'd like to be able to verify that the savepoint IDs are the same
+			# in both queries, but I'm not sure how.  Cross fingers!
+			expect_query('ROLLBACK TO "faff"')
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["wibble"])
+			expect_query("ROLLBACK")
+			in_transaction do |txn|
+				txn.insert("foo", :bar => 'wombat') do
+					txn.savepoint do
+						txn.insert("foo", :bar => 'baz') do
+							txn.rollback  # Just to show this *doesn't* happen
+						end
+					end.errback do
+						txn.insert("foo", :bar => 'wibble') do
+							txn.commit
+						end
+					end
+				end
+			end
+		end
+	end
+
+	it "handles nested SAVEPOINTs correctly" do
+		in_em do
+			expect(SecureRandom).to receive(:uuid).and_return("faff1")
+			expect(SecureRandom).to receive(:uuid).and_return("faff2")
+			expect_query("BEGIN")
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wombat"])
+			expect_query('SAVEPOINT "faff1"')
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["baz"])
+			# I'd like to be able to verify that the savepoint IDs are the same
+			# in both queries, but I'm not sure how.  Cross fingers!
+			expect_query('ROLLBACK TO "faff1"')
+			
+			expect_query('SAVEPOINT "faff2"')
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["wibble"])
+			expect_query('ROLLBACK TO "faff2"')
+			expect_query("COMMIT")
+
+			in_transaction do |txn|
+				txn.insert("foo", :bar => 'wombat') do
+					txn.savepoint do
+						txn.insert("foo", :bar => 'baz') do
+							txn.rollback  # Just to show this *doesn't* happen
+						end
+					end.errback do
+						txn.savepoint do
+							txn.insert("foo", :bar => 'wibble') do
+								txn.rollback  # Another "shouldn't happen"
+							end.errback do
+								txn.commit
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/spec/db_transaction_spec.rb

@@ -33,6 +33,20 @@ describe "PG::EM::Client::Helper#db_transaction" do
 		end
 	end
 
+	it "fails the transaction if COMMIT fails" do
+		dbl = double
+		expect(dbl).to receive(:foo)
+		expect(dbl).to_not receive(:bar)
+
+		in_em do
+			expect_query("BEGIN")
+			expect_query_failure("COMMIT")
+			in_transaction do |txn|
+				txn.commit
+			end.callback { dbl.bar }.errback { dbl.foo }
+		end
+	end
+
 	it "runs a simple INSERT correctly" do
 		in_em do
 			expect_query("BEGIN")
@@ -219,4 +233,21 @@ describe "PG::EM::Client::Helper#db_transaction" do
 			end
 		end
 	end
+
+	it "doesn't rollback back after a failed INSERT with autorollback = false" do
+		in_em do
+			expect_query("BEGIN")
+			expect_query_failure('INSERT INTO "foo" ("bar") VALUES ($1)', ["baz"])
+			expect_query('INSERT INTO "foo" ("bar") VALUES ($1)', ["wombat"])
+			expect_query("COMMIT")
+			in_transaction do |txn|
+				txn.autorollback_on_error = false
+				txn.insert("foo", :bar => 'baz').errback do
+					txn.insert("foo", :bar => 'wombat') do
+						txn.commit
+					end
+				end
+			end
+		end
+	end
 end

data/spec/txn_helper.rb

@@ -23,9 +23,16 @@ module TxnHelper
 	end
 
 	def in_em
-		EM.run do
-			EM.add_timer(0.5) { EM.stop; raise "test timeout" }
-			yield
+		begin
+			Timeout.timeout(0.5) do
+				EM.run do
+					yield
+				end
+			end
+		rescue Timeout::Error
+			EM.stop if EM.reactor_running?
+			raise RuntimeError,
+			      "EM test time exceeded"
 		end
 	end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: em-pg-client-helper
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-03 00:00:00.000000000 Z
+date: 2015-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-pg-client
@@ -230,7 +230,9 @@ files:
 - lib/em-pg-client-helper.rb
 - lib/em-pg-client-helper/deferrable_group.rb
 - lib/em-pg-client-helper/transaction.rb
+- spec/db_bulk_insert_spec.rb
 - spec/db_insert_spec.rb
+- spec/db_transaction_savepoint_spec.rb
 - spec/db_transaction_sequel_spec.rb
 - spec/db_transaction_spec.rb
 - spec/db_transaction_upsert_spec.rb