embulk-output-jdbc 0.8.3 → 0.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c01d9d1c8f2babb4d908929c0118a5a9c28d87f
-  data.tar.gz: c16e74184b0f5a59b8cac366bcfca99e5a38cb1c
+  metadata.gz: 9ac7ae92de373b40dfa5153d7d3e0e3bf3f99600
+  data.tar.gz: eea167726affb1c31c282cc9aee5c2b45c445d67
 SHA512:
-  metadata.gz: 5a3b33623d041ced79d8c4cb885ddbe8505da46395cb1953523874e58b4eb8a630eed4e9ca8a6c16eb4051bc44dd2c973dbb6cad3f6023daebddf8892eabaffa
-  data.tar.gz: 3df9712d68feca663e6bcbf053c8856ee2dc8779a80515da63457040a7bf0261d1a018e50eac461d10dd0fe4ba408c8e20bd58a4bfefbc6d076f7926f187d79f
+  metadata.gz: 48698e261cc369c0c594a4f7e50c9768e22b1ed6e527669688c95ec6d5dc0e8b8e1f7c114f18e15a77e1066334a966aa5aecd756cc627b2e8da533aff00e635b
+  data.tar.gz: f872b344503f72a42428fe20437ca9528cee34ad66db9d402be58806bf1ca2c71d0fbb210cd80bf300f2717e589f0ea869d1ecb9b87109f3397d4a073acd1fa2

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -15,6 +15,7 @@ import java.net.URL;
 import java.nio.charset.Charset;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.sql.Statement;
 import java.sql.Types;
 import java.sql.ResultSet;
 import java.sql.DatabaseMetaData;
@@ -1200,12 +1201,18 @@ public abstract class AbstractJdbcOutputPlugin
         protected void retryColumnsSetters() throws IOException, SQLException
         {
             int size = columnVisitors.size();
+            int[] updateCounts = batch.getLastUpdateCounts();
+            int index = 0;
             for (Record record : pageReader.getReadRecords()) {
-                for (int i = 0; i < size; i++) {
-                    ColumnSetterVisitor columnVisitor = new ColumnSetterVisitor(record, columnSetters.get(i));
-                    columns.get(i).visit(columnVisitor);
+                // retry failed records
+                if (index >= updateCounts.length || updateCounts[index] == Statement.EXECUTE_FAILED) {
+                    for (int i = 0; i < size; i++) {
+                        ColumnSetterVisitor columnVisitor = new ColumnSetterVisitor(record, columnSetters.get(i));
+                        columns.get(i).visit(columnVisitor);
+                    }
+                    batch.add();
                 }
-                batch.add();
+                index++;
             }
         }
     }

data/src/main/java/org/embulk/output/jdbc/BatchInsert.java

@@ -18,6 +18,9 @@ public interface BatchInsert
 
     public void flush() throws IOException, SQLException;
 
+    // should be implemented for retry
+    public int[] getLastUpdateCounts();
+
     public void finish() throws IOException, SQLException;
 
     public void setNull(int sqlType) throws IOException, SQLException;

data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java

@@ -3,6 +3,7 @@ package org.embulk.output.jdbc;
 import java.util.Calendar;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.sql.BatchUpdateException;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.Date;
@@ -26,6 +27,7 @@ public class StandardBatchInsert
     private int batchWeight;
     private int batchRows;
     private long totalRows;
+    private int[] lastUpdateCounts;
 
     public StandardBatchInsert(JdbcOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
@@ -70,17 +72,24 @@ public class StandardBatchInsert
 
     public void flush() throws IOException, SQLException
     {
+        lastUpdateCounts = new int[]{};
+
         if (batchWeight == 0) return;
 
         logger.info(String.format("Loading %,d rows", batchRows));
         long startTime = System.currentTimeMillis();
         try {
-            batch.executeBatch();  // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
+            lastUpdateCounts = batch.executeBatch();  // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
             double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
 
             totalRows += batchRows;
             logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
 
+        } catch (BatchUpdateException e) {
+            // will be used for retry
+            lastUpdateCounts = e.getUpdateCounts();
+            throw e;
+
         } finally {
             // clear for retry
             batch.clearBatch();
@@ -89,6 +98,12 @@ public class StandardBatchInsert
         }
     }
 
+    @Override
+    public int[] getLastUpdateCounts()
+    {
+        return lastUpdateCounts;
+    }
+
     public void finish() throws IOException, SQLException
     {
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.8.3
+  version: 0.8.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-07 00:00:00.000000000 Z
+date: 2019-05-08 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,7 +19,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.3.jar
+- classpath/embulk-output-jdbc-0.8.4.jar
 - lib/embulk/output/jdbc.rb
 - src/main/java/org/embulk/output/JdbcOutputPlugin.java
 - src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputConnector.java