embulk-output-jdbc 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55c72f27b523c6e9e8a9f4bdcc6e34238fc2ab93
-  data.tar.gz: 1d6350d92b143a0423c2b5de300853b2f92ca9e1
+  metadata.gz: 050247b8f08b7c7d9c34d2e0c36a4cf9dcfadc18
+  data.tar.gz: 19caf0641d799f247059ac7d5e3247e9895f4a02
 SHA512:
-  metadata.gz: d625cc29b72e815c187745ee8553761bf3e27408c98c3ce69cb7539810ad5b32dc390e3e6557cd65954c0102cbb02569b1031f85d4c6ebd50d85d27abb32ed86
-  data.tar.gz: f8adc582282ef5f15e71401da3650162cf965dd7b7ae38145b7139dfa8dcbfd49e241eb45acec9a43a122ce45371127934bed643f4edc891bf8af4c5bdcf9da0
+  metadata.gz: 2ea08bdeefad0859f036b475c0e13ea6c803c5d0c14a8ee8368ce912a6d0c743ba16fd460eb9ff6642f3ebb54cbaab798b1c52f757d4da77efd6034a4c5de9e0
+  data.tar.gz: 04c12e62081dc8938a57d7a1e12574235a1fb67eb6042a19408baaa2a6883e7400542a6909761b5d64b219bb214bed056de342409dd3b1bebb5c310ff9efeb9b

data/src/main/java/org/embulk/output/JdbcOutputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.output;
 
-import java.util.List;
 import java.util.Properties;
 import java.sql.Driver;
 import java.io.IOException;
@@ -13,11 +12,7 @@ import com.google.common.collect.ImmutableSet;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
-import org.embulk.output.jdbc.BatchInsert;
-import org.embulk.output.jdbc.StandardBatchInsert;
-import org.embulk.output.jdbc.JdbcOutputConnector;
-import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.*;
 
 public class JdbcOutputPlugin
         extends AbstractJdbcOutputPlugin
@@ -132,8 +127,8 @@ public class JdbcOutputPlugin
     }
 
     @Override
-    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
-        return new StandardBatchInsert(getConnector(task, true), mergeKeys);
+        return new StandardBatchInsert(getConnector(task, true), mergeConfig);
     }
 }

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -14,6 +14,7 @@ import java.sql.ResultSet;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
 
+import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 import org.joda.time.DateTimeZone;
 
@@ -91,6 +92,22 @@ public abstract class AbstractJdbcOutputPlugin
         @ConfigDefault("\"UTC\"")
         public DateTimeZone getDefaultTimeZone();
 
+        @Config("retry_limit")
+        @ConfigDefault("12")
+        public int getRetryLimit();
+
+        @Config("retry_wait")
+        @ConfigDefault("1000")
+        public int getRetryWait();
+
+        @Config("max_retry_wait")
+        @ConfigDefault("1800000") // 30 * 60 * 1000
+        public int getMaxRetryWait();
+
+        @Config("merge_rule")
+        @ConfigDefault("null")
+        public Optional<List<String>> getMergeRule();
+
         public void setActualTable(String actualTable);
         public String getActualTable();
 
@@ -213,7 +230,7 @@ public abstract class AbstractJdbcOutputPlugin
 
     protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
 
-    protected abstract BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException;
+    protected abstract BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException;
 
     protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
             boolean autoCommit) throws SQLException
@@ -357,7 +374,7 @@ public abstract class AbstractJdbcOutputPlugin
             final Schema schema, final int taskCount)
     {
         try {
-            withRetry(new IdempotentSqlRunnable() {  // no intermediate data if isDirectModify == true
+            withRetry(task, new IdempotentSqlRunnable() {  // no intermediate data if isDirectModify == true
                 public void run() throws SQLException
                 {
                     JdbcOutputConnection con = newConnection(task, true, false);
@@ -379,7 +396,7 @@ public abstract class AbstractJdbcOutputPlugin
     {
         if (!task.getMode().isDirectModify()) {  // no intermediate data if isDirectModify == true
             try {
-                withRetry(new IdempotentSqlRunnable() {
+                withRetry(task, new IdempotentSqlRunnable() {
                     public void run() throws SQLException
                     {
                         JdbcOutputConnection con = newConnection(task, false, false);
@@ -405,7 +422,7 @@ public abstract class AbstractJdbcOutputPlugin
 
         if (!task.getMode().isDirectModify()) {  // no intermediate data if isDirectModify == true
             try {
-                withRetry(new IdempotentSqlRunnable() {
+                withRetry(task, new IdempotentSqlRunnable() {
                     public void run() throws SQLException
                     {
                         JdbcOutputConnection con = newConnection(task, true, true);
@@ -682,7 +699,7 @@ public abstract class AbstractJdbcOutputPlugin
             if (task.getNewTableSchema().isPresent()) {
                 con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
             }
-            con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(), task.getMergeKeys().get());
+            con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(), new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()));
             break;
 
         case REPLACE:
@@ -829,10 +846,11 @@ public abstract class AbstractJdbcOutputPlugin
         // instantiate BatchInsert without table name
         BatchInsert batch = null;
         try {
-            batch = newBatchInsert(task,
-                    task.getMode() == Mode.MERGE_DIRECT ?
-                        task.getMergeKeys() :
-                        Optional.<List<String>>absent());
+            Optional<MergeConfig> config = Optional.absent();
+            if (task.getMode() == Mode.MERGE_DIRECT) {
+                config = Optional.of(new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()));
+            }
+            batch = newBatchInsert(task, config);
         } catch (IOException | SQLException ex) {
             throw new RuntimeException(ex);
         }
@@ -861,7 +879,7 @@ public abstract class AbstractJdbcOutputPlugin
             }
             batch.prepare(destTable, insertIntoSchema);
 
-            PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
+            PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize(), task);
             batch = null;
             return output;
 
@@ -879,7 +897,7 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
-    public static class PluginPageOutput
+    public class PluginPageOutput
             implements TransactionalPageOutput
     {
         protected final List<Column> columns;
@@ -888,10 +906,11 @@ public abstract class AbstractJdbcOutputPlugin
         private final BatchInsert batch;
         private final int batchSize;
         private final int forceBatchFlushSize;
+        private final PluginTask task;
 
         public PluginPageOutput(final PageReader pageReader,
                 BatchInsert batch, List<ColumnSetter> columnSetters,
-                int batchSize)
+                int batchSize, PluginTask task)
         {
             this.pageReader = pageReader;
             this.batch = batch;
@@ -904,6 +923,7 @@ public abstract class AbstractJdbcOutputPlugin
                             }
                         }));
             this.batchSize = batchSize;
+            this.task = task;
             this.forceBatchFlushSize = batchSize * 2;
         }
 
@@ -914,15 +934,33 @@ public abstract class AbstractJdbcOutputPlugin
                 pageReader.setPage(page);
                 while (pageReader.nextRecord()) {
                     if (batch.getBatchWeight() > forceBatchFlushSize) {
-                        batch.flush();
+                        withRetry(task, new IdempotentSqlRunnable() {
+                            @Override
+                            public void run() throws SQLException {
+                                try {
+                                    batch.flush();
+                                } catch (IOException ex) {
+                                    throw new RuntimeException(ex);
+                                }
+                            }
+                        });
                     }
                     handleColumnsSetters();
                     batch.add();
                 }
                 if (batch.getBatchWeight() > batchSize) {
-                    batch.flush();
+                    withRetry(task, new IdempotentSqlRunnable() {
+                        @Override
+                        public void run() throws SQLException {
+                            try {
+                                batch.flush();
+                            } catch (IOException ex) {
+                                throw new RuntimeException(ex);
+                            }
+                        }
+                    });
                 }
-            } catch (IOException | SQLException ex) {
+            } catch (IOException | SQLException | InterruptedException ex) {
                 throw new RuntimeException(ex);
             }
         }
@@ -931,8 +969,17 @@ public abstract class AbstractJdbcOutputPlugin
         public void finish()
         {
             try {
-                batch.finish();
-            } catch (IOException | SQLException ex) {
+                withRetry(task, new IdempotentSqlRunnable() {
+                    @Override
+                    public void run() throws SQLException {
+                        try {
+                            batch.finish();
+                        } catch (IOException ex) {
+                            throw new RuntimeException(ex);
+                        }
+                    }
+                });
+            } catch (InterruptedException | SQLException ex) {
                 throw new RuntimeException(ex);
             }
         }
@@ -967,73 +1014,33 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
+    protected boolean isRetryableException(SQLException exception)
+    {
+        return isRetryableException(exception.getSQLState(), exception.getErrorCode());
+    }
+    protected boolean isRetryableException(String sqlState, int errorCode)
+    {
+        return false;
+    }
+
+
     public static interface IdempotentSqlRunnable
     {
         public void run() throws SQLException;
     }
 
-    protected void withRetry(IdempotentSqlRunnable op)
+    protected void withRetry(PluginTask task, IdempotentSqlRunnable op)
             throws SQLException, InterruptedException
     {
-        withRetry(op, "Operation failed");
+        withRetry(task, op, "Operation failed");
     }
 
-    protected void withRetry(final IdempotentSqlRunnable op, final String errorMessage)
+    protected void withRetry(PluginTask task, final IdempotentSqlRunnable op, final String errorMessage)
             throws SQLException, InterruptedException
     {
         try {
-            retryExecutor()
-                .withRetryLimit(12)
-                .withInitialRetryWait(1000)
-                .withMaxRetryWait(30 * 60 * 1000)
-                .runInterruptible(new Retryable<Void>() {
-                    public Void call() throws Exception
-                    {
-                        op.run();
-                        return null;
-                    }
-
-                    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                    {
-                        if (exception instanceof SQLException) {
-                            SQLException ex = (SQLException) exception;
-                            String sqlState = ex.getSQLState();
-                            int errorCode = ex.getErrorCode();
-                            logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
-                                    errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
-                                    buildExceptionMessage(exception));
-                        } else {
-                            logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
-                                    errorMessage, retryCount, retryLimit, retryWait/1000,
-                                    buildExceptionMessage(exception));
-                        }
-                        if (retryCount % 3 == 0) {
-                            logger.info("Error details:", exception);
-                        }
-                    }
-
-                    public void onGiveup(Exception firstException, Exception lastException)
-                    {
-                        if (firstException instanceof SQLException) {
-                            SQLException ex = (SQLException) firstException;
-                            String sqlState = ex.getSQLState();
-                            int errorCode = ex.getErrorCode();
-                            logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
-                        }
-                    }
-
-                    public boolean isRetryableException(Exception exception)
-                    {
-                        //if (exception instanceof SQLException) {
-                        //    SQLException ex = (SQLException) exception;
-                        //    String sqlState = ex.getSQLState();
-                        //    int errorCode = ex.getErrorCode();
-                        //    return isRetryableSQLException(ex);
-                        //}
-                        return false;  // TODO
-                    }
-                });
-
+            buildRetryExecutor(task)
+                .runInterruptible(new RetryableSQLExecution(op, errorMessage));
         } catch (ExecutionException ex) {
             Throwable cause = ex.getCause();
             Throwables.propagateIfInstanceOf(cause, SQLException.class);
@@ -1041,24 +1048,87 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
-    private String buildExceptionMessage(Throwable ex) {
-        StringBuilder sb = new StringBuilder();
-        sb.append(ex.getMessage());
-        if (ex.getCause() != null) {
-            buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
-        }
-        return sb.toString();
+    private static RetryExecutor buildRetryExecutor(PluginTask task) {
+        return retryExecutor()
+                .withRetryLimit(task.getRetryLimit())
+                .withInitialRetryWait(task.getRetryWait())
+                .withMaxRetryWait(task.getMaxRetryWait());
     }
 
-    private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
-        if (!lastMessage.equals(ex.getMessage())) {
-            // suppress same messages
-            sb.append(" < ");
+    class RetryableSQLExecution implements Retryable<Void> {
+        private final String errorMessage;
+        private final IdempotentSqlRunnable op;
+
+        private final Logger logger = Exec.getLogger(this.getClass());
+
+        public RetryableSQLExecution(IdempotentSqlRunnable op, String errorMessage) {
+            this.errorMessage = errorMessage;
+            this.op = op;
+        }
+
+        public Void call() throws Exception {
+            op.run();
+            return null;
+        }
+
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+        {
+            if (exception instanceof SQLException) {
+                SQLException ex = (SQLException) exception;
+                String sqlState = ex.getSQLState();
+                int errorCode = ex.getErrorCode();
+                logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
+                        errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
+                        buildExceptionMessage(exception));
+            } else {
+                logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
+                        errorMessage, retryCount, retryLimit, retryWait/1000,
+                        buildExceptionMessage(exception));
+            }
+            if (retryCount % 3 == 0) {
+                logger.info("Error details:", exception);
+            }
+        }
+
+        public void onGiveup(Exception firstException, Exception lastException)
+        {
+            if (firstException instanceof SQLException) {
+                SQLException ex = (SQLException) firstException;
+                String sqlState = ex.getSQLState();
+                int errorCode = ex.getErrorCode();
+                logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
+            }
+        }
+
+        public boolean isRetryableException(Exception exception) {
+            if (exception instanceof SQLException) {
+                SQLException ex = (SQLException)exception;
+
+                return AbstractJdbcOutputPlugin.this.isRetryableException(ex);
+            } else {
+                return false;
+            }
+        }
+
+        private String buildExceptionMessage(Throwable ex) {
+            StringBuilder sb = new StringBuilder();
             sb.append(ex.getMessage());
+            if (ex.getCause() != null) {
+                buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
+            }
+            return sb.toString();
         }
-        if (ex.getCause() == null) {
-            return;
+
+        private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
+            if (!lastMessage.equals(ex.getMessage())) {
+                // suppress same messages
+                sb.append(" < ");
+                sb.append(ex.getMessage());
+            }
+            if (ex.getCause() == null) {
+                return;
+            }
+            buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
         }
-        buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
     }
 }

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -9,6 +9,7 @@ import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+
 import org.slf4j.Logger;
 import com.google.common.base.Optional;
 import org.embulk.spi.Exec;
@@ -240,11 +241,11 @@ public class JdbcOutputConnection
         return ColumnDeclareType.SIMPLE;
     }
 
-    public PreparedStatement prepareBatchInsertStatement(String toTable, JdbcSchema toTableSchema, Optional<List<String>> mergeKeys) throws SQLException
+    public PreparedStatement prepareBatchInsertStatement(String toTable, JdbcSchema toTableSchema, Optional<MergeConfig> mergeConfig) throws SQLException
     {
         String sql;
-        if (mergeKeys.isPresent()) {
-            sql = buildPreparedMergeSql(toTable, toTableSchema, mergeKeys.get());
+        if (mergeConfig.isPresent()) {
+            sql = buildPreparedMergeSql(toTable, toTableSchema, mergeConfig.get());
         } else {
             sql = buildPreparedInsertSql(toTable, toTableSchema);
         }
@@ -274,7 +275,7 @@ public class JdbcOutputConnection
         return sb.toString();
     }
 
-    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, List<String> mergeKeys) throws SQLException
+    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
     {
         throw new UnsupportedOperationException("not implemented");
     }
@@ -334,11 +335,11 @@ public class JdbcOutputConnection
         return sb.toString();
     }
 
-    protected void collectMerge(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    protected void collectMerge(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {
         Statement stmt = connection.createStatement();
         try {
-            String sql = buildCollectMergeSql(fromTables, schema, toTable, mergeKeys);
+            String sql = buildCollectMergeSql(fromTables, schema, toTable, mergeConfig);
             executeUpdate(stmt, sql);
             commitIfNecessary(connection);
         } catch (SQLException ex) {
@@ -348,7 +349,7 @@ public class JdbcOutputConnection
         }
     }
 
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {
         throw new UnsupportedOperationException("not implemented");
     }

data/src/main/java/org/embulk/output/jdbc/MergeConfig.java

@@ -0,0 +1,23 @@
+package org.embulk.output.jdbc;
+
+import com.google.common.base.Optional;
+
+import java.util.List;
+
+public class MergeConfig {
+    private final List<String> mergeKeys;
+    private final Optional<List<String>> mergeRule;
+
+    public MergeConfig(List<String> mergeKeys, Optional<List<String>> mergeRule) {
+        this.mergeKeys = mergeKeys;
+        this.mergeRule = mergeRule;
+    }
+
+    public List<String> getMergeKeys() {
+        return mergeKeys;
+    }
+
+    public Optional<List<String>> getMergeRule() {
+        return mergeRule;
+    }
+}

data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java

@@ -19,7 +19,7 @@ public class StandardBatchInsert
     private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
 
     private final JdbcOutputConnector connector;
-    private final Optional<List<String>> mergeKeys;
+    private final Optional<MergeConfig> mergeConfig;
 
     private JdbcOutputConnection connection;
     private PreparedStatement batch;
@@ -28,10 +28,10 @@ public class StandardBatchInsert
     private int batchRows;
     private long totalRows;
 
-    public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    public StandardBatchInsert(JdbcOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
         this.connector = connector;
-        this.mergeKeys = mergeKeys;
+        this.mergeConfig = mergeConfig;
     }
 
     public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
@@ -46,7 +46,7 @@ public class StandardBatchInsert
 
     protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
     {
-        return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
+        return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeConfig);
     }
 
     public int getBatchWeight()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-23 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -28,6 +28,7 @@ files:
 - src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java
 - src/main/java/org/embulk/output/jdbc/JdbcSchema.java
 - src/main/java/org/embulk/output/jdbc/JdbcUtils.java
+- src/main/java/org/embulk/output/jdbc/MergeConfig.java
 - src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java
 - src/main/java/org/embulk/output/jdbc/TimestampFormat.java
 - src/main/java/org/embulk/output/jdbc/ToString.java
@@ -58,7 +59,7 @@ files:
 - src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
 - src/test/java/org/embulk/output/TimestampFormatTest.java
 - src/test/java/org/embulk/output/tester/EmbulkPluginTester.java
-- classpath/embulk-output-jdbc-0.6.1.jar
+- classpath/embulk-output-jdbc-0.6.2.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0