em-pg-client-helper 2.0.1 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f805f782a78651db5a93dbcba7516e3e1b0abe4f
-  data.tar.gz: 72132229cb096de84eb4c823e2d49b1d5691cc40
+  metadata.gz: 5d0deea932297d54ea2850f086680cd7621f5402
+  data.tar.gz: ffbb8fa419655a46c072799871bda278a815559a
 SHA512:
-  metadata.gz: 3a8390c30559d4986d5a9704ea4c118b5948cb87811c6208e6399bd874ee2a5979c1b3690b78684be69c3f6e77b28fa93f06c2bb6e9de6cffe52b0756b1b0bd5
-  data.tar.gz: b6efc7278ca5a06be0930cfa7a0a5125e5345f6215d27fa724e2d59fd2897700cf0e2bd679fd1d8eebc7fb30287722b5f285179941996361354757747df1de9a
+  metadata.gz: b9682d5cc639060a96e73f4e061987400bb1a70335bb74eb4c9f4bc3f391500721705a32e6a0d492bf3826fbb3dceb2728359d7e1a9c5144fa94feb893593ffd
+  data.tar.gz: 9b78c7ab701b04ae9ecb1ab34f0ed8981500c61d9b1a8ced8ca5aabf396ad3db5bd1031ddad8a1ed5b9f6354d755d53a557339684d820a78f39d4a90c24a2972

data/lib/em-pg-client-helper/transaction.rb

@@ -238,31 +238,54 @@ class PG::EM::Client::Helper::Transaction
 	# @since 2.0.0
 	#
 	def bulk_insert(tbl, columns, rows, &blk)
-		if rows.length > 1000
-			bulk_insert(tbl, columns, rows[0..999]) do |count1|
-				bulk_insert(tbl, columns, rows[1000..-1]) do |count2|
-					blk.call(count1 + count2)
+		columns.map!(&:to_sym)
+
+		EM::Completion.new.tap do |df|
+			@dg.add(df)
+			df.callback(&blk) if blk
+
+			unique_index_columns_for_table(tbl) do |indexes|
+				# Guh hand-hacked SQL is fugly... but what I'm doing is so utterly
+				# niche that Sequel doesn't support it.
+				q_tbl = usdb.literal(tbl.to_sym)
+				q_cols = columns.map { |c| usdb.literal(c) }
+
+				# If there are any unique indexes which the set of columns to
+				# be inserted into *doesn't* completely cover, we need to error
+				# out, because that will cause no rows (or, at most, one row) to
+				# be successfully inserted.  In *theory*, a unique index with all-but-one
+				# row inserted *could* work, but that would only work if every value
+				# inserted was different, but quite frankly, I think that's a wicked
+				# corner case I'm not going to go *anywhere* near.
+				#
+				unless indexes.all? { |i| (i - columns).empty? }
+					ex = ArgumentError.new("Unique index columns not covered by data columns")
+					if @autorollback_on_error
+						df.fail(ex)
+						rollback(ex)
+					else
+						df.fail(ex)
+					end
+				else
+					subselect_where = indexes.map do |idx|
+						"(" + idx.map do |c|
+							"src.#{usdb.literal(c)}=dst.#{usdb.literal(c)}"
+						end.join(" AND ") + ")"
+					end.join(" OR ")
+
+					subselect = "SELECT 1 FROM #{q_tbl} AS dst WHERE #{subselect_where}"
+
+					vals = rows.map do |row|
+						"(" + row.map { |v| usdb.literal(v) }.join(", ") + ")"
+					end.join(", ")
+					q = "INSERT INTO #{q_tbl} (SELECT * FROM (VALUES #{vals}) " +
+						 "AS src (#{q_cols.join(", ")}) WHERE NOT EXISTS (#{subselect}))"
+					exec(q) do |res|
+						df.succeed(res.cmd_tuples)
+					end
 				end
-			end
-		else
-			# Guh hand-hacked SQL is fugly... but what I'm doing is so utterly
-			# niche that Sequel doesn't support it.
-			q_tbl = mock_db.literal(tbl.to_sym)
-			q_cols = columns.map { |c| mock_db.literal(c.to_sym) }
-
-			subselect = "SELECT 1 FROM #{q_tbl} AS dst WHERE " +
-							q_cols.map { |c| "src.#{c}=dst.#{c}" }.join(" AND ")
-
-			vals = rows.map do |row|
-				"(" + row.map { |v| mock_db.literal(v) }.join(", ") + ")"
-			end.join(", ")
-			q = "INSERT INTO #{q_tbl} (SELECT * FROM (VALUES #{vals}) " +
-				 "AS src (#{q_cols.join(", ")}) WHERE NOT EXISTS (#{subselect}))"
-			exec(q).tap do |df|
-				df.callback do |res|
-					df.succeed(res.cmd_tuples)
-				end
-				df.callback(&blk) if blk
+			end.errback do |ex|
+				df.fail(ex)
 			end
 		end
 	end
@@ -293,12 +316,13 @@ class PG::EM::Client::Helper::Transaction
 	#   specific query finishes.
 	#
 	def exec(sql, values=[], &blk)
+		trace_query(sql, values)
+
 		if @finished
 			raise ClosedError,
 			      "Cannot execute a query in a transaction that has been closed"
 		end
 
-		trace_query(sql, values)
 		@conn.exec_defer(sql, values).tap do |df|
 			@dg.add(df)
 			df.callback(&blk) if blk
@@ -316,7 +340,41 @@ class PG::EM::Client::Helper::Transaction
 		$stderr.puts "#{@conn.inspect}: #{q} #{v.inspect}" if ENV['EM_PG_TXN_TRACE']
 	end
 
-	def mock_db
-		@mock_db ||= Sequel.connect("mock://postgres")
+	# The Utility Sequel Data Base.  A mock PgSQL Sequel database that we can
+	# call methods like `#literal` on, for easy quoting.
+	#
+	def usdb
+		@usdb ||= Sequel.connect("mock://postgres")
+	end
+
+	# Find the unique indexes for a table, and yield the columns in each.
+	#
+	# This is used to work out what sets of fields to match on when doing
+	# bulk inserts.
+	#
+	# @yield [Array<Array<Symbol>>] Each element in the yielded array is a
+	#   list of the fields which make up a single unique index.
+	#
+	def unique_index_columns_for_table(t)
+		q = "SELECT a.attrelid AS idxid,
+		            a.attname  AS name,
+		            (SELECT substring(pg_catalog.pg_get_expr(d.adbin, d.adrelid) for 128)
+		               FROM pg_catalog.pg_attrdef d
+		              WHERE d.adrelid = a.attrelid AND d.adnum = a.attnum AND a.atthasdef) AS default
+		       FROM pg_catalog.pg_attribute AS a
+		       JOIN pg_catalog.pg_index AS i ON a.attrelid=i.indexrelid
+		       JOIN pg_catalog.pg_class AS c ON c.oid=i.indrelid
+		      WHERE c.relname=#{usdb.literal(t.to_s)}
+		        AND a.attnum > 0
+		        AND NOT a.attisdropped
+		        AND i.indisunique"
+
+		exec(q) do |res|
+			yield(res.to_a.each_with_object(Hash.new { |h,k| h[k] = [] }) do |r, h|
+				# Skip auto-incrementing fields; they can take care of themselves
+				next if r["default"] =~ /^nextval/
+				h[r["idxid"]] << r["name"].to_sym
+			end.values)
+		end
 	end
 end

data/spec/db_bulk_insert_spec.rb

@@ -2,26 +2,43 @@ require_relative './spec_helper'
 
 describe "PG::EM::Client::Helper#db_bulk_insert" do
 	let(:mock_conn) { double(PG::EM::Client) }
+	let(:fast_query) do
+		'INSERT INTO "foo" ' +
+		'(SELECT * FROM (VALUES (1, \'x\'), (3, \'y\')) ' +
+		'AS src ("bar", "baz") ' +
+		'WHERE NOT EXISTS ' +
+		'(SELECT 1 FROM "foo" AS dst ' +
+		'WHERE (src."bar"=dst."bar" AND src."baz"=dst."baz")))'
+	end
+
+	def expect_unique_indexes_query(indexes)
+		indexes.map! do |i|
+			idxid = rand(1000000).to_s
+			i.map { |n| {"idxid" => idxid, "name" => n.to_s} }
+		end
+		indexes.flatten!
+
+		expect_query(/^SELECT a\.attrelid/,
+		             [], 0.001, :succeed, indexes
+		            )
+	end
 
 	it "inserts multiple records" do
 		expect(dbl = double).to receive(:results).with(2)
+		expect(dbl).to_not receive(:errback)
 
 		in_em do
 			expect_query("BEGIN")
-		   expect_query('INSERT INTO "foo" ' +
-		                '(SELECT * FROM (VALUES (1, \'x\'), (3, \'y\')) ' +
-		                'AS src ("bar", "baz") ' +
-		                'WHERE NOT EXISTS ' +
-		                '(SELECT 1 FROM "foo" AS dst ' +
-		                'WHERE src."bar"=dst."bar" AND src."baz"=dst."baz"))',
-		                [], 0.001, :succeed, Struct.new(:cmd_tuples).new(2)
-		               )
-		   expect_query("COMMIT")
+			expect_unique_indexes_query([[:bar, :baz]])
+			expect_query(fast_query,
+			             [], 0.001, :succeed, Struct.new(:cmd_tuples).new(2)
+			            )
+			expect_query("COMMIT")
 
 			db_bulk_insert(mock_conn, "foo", [:bar, :baz], [[1, "x"], [3, "y"]]) do |count|
 				dbl.results(count)
 				EM.stop
-			end
+			end.errback { dbl.errback; EM.stop }
 		end
 	end
 end

data/spec/db_transaction_spec.rb

@@ -35,15 +35,15 @@ describe "PG::EM::Client::Helper#db_transaction" do
 
 	it "fails the transaction if COMMIT fails" do
 		dbl = double
-		expect(dbl).to receive(:foo)
-		expect(dbl).to_not receive(:bar)
+		expect(dbl).to receive(:errback)
+		expect(dbl).to_not receive(:callback)
 
 		in_em do
 			expect_query("BEGIN")
 			expect_query_failure("COMMIT")
 			in_transaction do |txn|
 				txn.commit
-			end.callback { dbl.bar }.errback { dbl.foo }
+			end.callback { dbl.callback }.errback { dbl.errback }
 		end
 	end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: em-pg-client-helper
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.0.2
 platform: ruby
 authors:
 - Matt Palmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-15 00:00:00.000000000 Z
+date: 2015-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-pg-client