embulk-output-jdbc 0.2.4 → 0.3.0

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java

@@ -1,8 +1,8 @@
-package org.embulk.output.jdbc;
-
-import java.sql.SQLException;
-
-public interface JdbcOutputConnector
-{
-    public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
-}
+package org.embulk.output.jdbc;
+
+import java.sql.SQLException;
+
+public interface JdbcOutputConnector
+{
+    public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
+}

data/src/main/java/org/embulk/output/jdbc/JdbcSchema.java

@@ -1,37 +1,60 @@
-package org.embulk.output.jdbc;
-
-import java.util.List;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
-
-public class JdbcSchema
-{
-    private List<JdbcColumn> columns;
-
-    @JsonCreator
-    public JdbcSchema(List<JdbcColumn> columns)
-    {
-        this.columns = columns;
-    }
-
-    @JsonValue
-    public List<JdbcColumn> getColumns()
-    {
-        return columns;
-    }
-
-    public int getCount()
-    {
-        return columns.size();
-    }
-
-    public JdbcColumn getColumn(int i)
-    {
-        return columns.get(i);
-    }
-
-    public String getColumnName(int i)
-    {
-        return columns.get(i).getName();
-    }
-}
+package org.embulk.output.jdbc;
+
+import java.util.List;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class JdbcSchema
+{
+    private List<JdbcColumn> columns;
+
+    @JsonCreator
+    public JdbcSchema(List<JdbcColumn> columns)
+    {
+        this.columns = columns;
+    }
+
+    @JsonValue
+    public List<JdbcColumn> getColumns()
+    {
+        return columns;
+    }
+
+    public Optional<JdbcColumn> findColumn(String name)
+    {
+        for (JdbcColumn column : columns) {
+            if (column.getName().equals(name)) {
+                return Optional.of(column);
+            }
+        }
+        return Optional.absent();
+    }
+
+    public int getCount()
+    {
+        return columns.size();
+    }
+
+    public JdbcColumn getColumn(int i)
+    {
+        return columns.get(i);
+    }
+
+    public String getColumnName(int i)
+    {
+        return columns.get(i).getName();
+    }
+
+    public static JdbcSchema filterSkipColumns(JdbcSchema schema)
+    {
+        ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
+        for (JdbcColumn c : schema.getColumns()) {
+            if (!c.isSkipColumn()) {
+                builder.add(c);
+            }
+        }
+        return new JdbcSchema(builder.build());
+    }
+}

data/src/main/java/org/embulk/output/jdbc/JdbcUtils.java

@@ -1,155 +1,155 @@
-package org.embulk.output.jdbc;
-
-import java.sql.Connection;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.slf4j.Logger;
-import org.embulk.spi.Exec;
-
-public class JdbcUtils
-{
-    public final Logger logger = Exec.getLogger(JdbcUtils.class.getName());
-
-    private static String[] SEARCH_STRING_SPECIAL_CHARS = new String[] { "_", "%" };
-
-    public static String escapeSearchString(String searchString, String escapeString)
-    {
-        if (searchString != null && escapeString != null) {
-            // First of all, escape escapeString '\' itself: '\' -> '\\'
-            searchString = searchString.replaceAll(Pattern.quote(escapeString),
-                    Matcher.quoteReplacement(escapeString + escapeString));
-            for (String specialChar : SEARCH_STRING_SPECIAL_CHARS) {
-                if (specialChar.equals(escapeString)) {
-                    throw new IllegalArgumentException("Special char " + specialChar + " cannot be an escape char");
-                }
-                searchString = searchString.replaceAll(Pattern.quote(specialChar),
-                        Matcher.quoteReplacement(escapeString + specialChar));
-            }
-        }
-        return searchString;
-    }
-
-    private Class<?> connectionClass;
-
-    // Connection.isValid() is available from Java 1.6 + JDBC4.
-    //private boolean hasIsValid;
-
-    // Connection.setNetworkTimeout() is available from Java 1.7 + JDBC4.
-    //private boolean hasSetNetworkTimeout;
-    //private Method setNetworkTimeoutMethod;
-
-    public JdbcUtils(Class<?> connectionClass)
-    {
-        this.connectionClass = connectionClass;
-
-        //configureSetNetworkTimeout();
-    }
-
-    public int executeUpdateWithSqlLogging(Statement stmt, String sql) throws SQLException
-    {
-        logger.info("SQL: " + sql);
-        long startTime = System.currentTimeMillis();
-        int count = stmt.executeUpdate(sql);
-        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
-        if (count == 0) {
-            logger.info(String.format("> %.2f seconds", seconds));
-        } else {
-            logger.info(String.format("> %.2f seconds (%,d rows)", seconds, count));
-        }
-        return count;
-    }
-
-    //private void configureSetNetworkTimeout() {
-    //    try {
-    //        Method m = connectionClass.getMethod("setNetworkTimeout", Executor.class, int.class);
-    //        if (isCallableMethod(m)) {
-    //            setNetworkTimeoutMethod = m;
-    //            hasSetNetworkTimeout = true;
-    //        }
-    //    } catch (SecurityException ex) {
-    //    } catch (NoSuchMethodException ex) {
-    //    }
-    //}
-
-    //private boolean isCallableMethod(Method m) {
-    //    int modifiers = m.getModifiers();
-    //    if (Modifier.isAbstract(modifiers)) {
-    //        // Method.invoke throws java.lang.AbstractMethodError if it's an
-    //        // abstract method. Applications can't catch AbstractMethodError.
-    //        return false;
-    //    }
-    //    if (!Modifier.isPublic(modifiers)) {
-    //        // we can only call public methods
-    //        return false;
-    //    }
-    //    return true;
-    //}
-
-    // PostgreSQL JDBC driver implements isValid() method. But the
-    // implementation throws following exception:
-    // "java.io.IOException: Method org.postgresql.jdbc4.Jdbc4Connection.isValid(int) is not yet implemented."
-    //
-    // So, checking mechanism doesn't work at all.
-    // Thus here just runs "SELECT 1" to check connectivity.
-    //
-    //public boolean isValidConnection(Connection connection, int timeout) throws SQLException
-    //{
-    //    Statement stmt = connection.createStatement();
-    //    try {
-    //        stmt.executeQuery("SELECT 1").close();
-    //        return true;
-    //    } catch (SQLException ex) {
-    //        return false;
-    //    } finally {
-    //        stmt.close();
-    //    }
-    //}
-
-    //public void setNetworkTimeout(Connection connection,
-    //        Executor executor, int milliseconds) throws SQLException {
-    //    Throwable exception = null;
-    //    if (hasSetNetworkTimeout) {
-    //        try {
-    //            setNetworkTimeoutMethod.invoke(connection, executor, milliseconds);
-    //            return;
-    //
-    //        } catch (IllegalArgumentException ex) {
-    //            // ignore error
-    //            LOG.warn("Connection.setNetworkTimeout failed due to IllegalArgumentException.");
-    //            exception = ex;
-    //
-    //        } catch (IllegalAccessException ex) {
-    //            // ignore error
-    //            LOG.warn("Connection.setNetworkTimeout failed due to IllegalAccessException.");
-    //            exception = ex;
-    //
-    //        } catch (InvocationTargetException ex) {
-    //            //Throwable cause = ex.getTargetException();
-    //            //if (cause instanceof SQLException) {
-    //            //    throw (SQLException) cause;
-    //            //} else if (cause instanceof RuntimeException) {
-    //            //    throw (RuntimeException) cause;
-    //            //} else if (cause instanceof Error) {
-    //            //    throw (Error) cause;
-    //            //} else {
-    //            //    throw new SQLException(cause);
-    //            //}
-    //            exception = ex.getTargetException();
-    //            // It's safe to ignore exceptions.
-    //        }
-    //
-    //        hasSetNetworkTimeout = false;
-    //    }
-    //
-    //    if (exception != null) {
-    //        LOG.warn("Connection.setNetworkTimeout is not available: "+exception);
-    //    } else {
-    //        LOG.warn("Connection.setNetworkTimeout is not available.");
-    //    }
-    //    // TODO any substitute implementations?
-    //}
-}
-
+package org.embulk.output.jdbc;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.embulk.spi.Exec;
+
+public class JdbcUtils
+{
+    public final Logger logger = Exec.getLogger(JdbcUtils.class.getName());
+
+    private static String[] SEARCH_STRING_SPECIAL_CHARS = new String[] { "_", "%" };
+
+    public static String escapeSearchString(String searchString, String escapeString)
+    {
+        if (searchString != null && escapeString != null) {
+            // First of all, escape escapeString '\' itself: '\' -> '\\'
+            searchString = searchString.replaceAll(Pattern.quote(escapeString),
+                    Matcher.quoteReplacement(escapeString + escapeString));
+            for (String specialChar : SEARCH_STRING_SPECIAL_CHARS) {
+                if (specialChar.equals(escapeString)) {
+                    throw new IllegalArgumentException("Special char " + specialChar + " cannot be an escape char");
+                }
+                searchString = searchString.replaceAll(Pattern.quote(specialChar),
+                        Matcher.quoteReplacement(escapeString + specialChar));
+            }
+        }
+        return searchString;
+    }
+
+    private Class<?> connectionClass;
+
+    // Connection.isValid() is available from Java 1.6 + JDBC4.
+    //private boolean hasIsValid;
+
+    // Connection.setNetworkTimeout() is available from Java 1.7 + JDBC4.
+    //private boolean hasSetNetworkTimeout;
+    //private Method setNetworkTimeoutMethod;
+
+    public JdbcUtils(Class<?> connectionClass)
+    {
+        this.connectionClass = connectionClass;
+
+        //configureSetNetworkTimeout();
+    }
+
+    public int executeUpdateWithSqlLogging(Statement stmt, String sql) throws SQLException
+    {
+        logger.info("SQL: " + sql);
+        long startTime = System.currentTimeMillis();
+        int count = stmt.executeUpdate(sql);
+        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+        if (count == 0) {
+            logger.info(String.format("> %.2f seconds", seconds));
+        } else {
+            logger.info(String.format("> %.2f seconds (%,d rows)", seconds, count));
+        }
+        return count;
+    }
+
+    //private void configureSetNetworkTimeout() {
+    //    try {
+    //        Method m = connectionClass.getMethod("setNetworkTimeout", Executor.class, int.class);
+    //        if (isCallableMethod(m)) {
+    //            setNetworkTimeoutMethod = m;
+    //            hasSetNetworkTimeout = true;
+    //        }
+    //    } catch (SecurityException ex) {
+    //    } catch (NoSuchMethodException ex) {
+    //    }
+    //}
+
+    //private boolean isCallableMethod(Method m) {
+    //    int modifiers = m.getModifiers();
+    //    if (Modifier.isAbstract(modifiers)) {
+    //        // Method.invoke throws java.lang.AbstractMethodError if it's an
+    //        // abstract method. Applications can't catch AbstractMethodError.
+    //        return false;
+    //    }
+    //    if (!Modifier.isPublic(modifiers)) {
+    //        // we can only call public methods
+    //        return false;
+    //    }
+    //    return true;
+    //}
+
+    // PostgreSQL JDBC driver implements isValid() method. But the
+    // implementation throws following exception:
+    // "java.io.IOException: Method org.postgresql.jdbc4.Jdbc4Connection.isValid(int) is not yet implemented."
+    //
+    // So, checking mechanism doesn't work at all.
+    // Thus here just runs "SELECT 1" to check connectivity.
+    //
+    //public boolean isValidConnection(Connection connection, int timeout) throws SQLException
+    //{
+    //    Statement stmt = connection.createStatement();
+    //    try {
+    //        stmt.executeQuery("SELECT 1").close();
+    //        return true;
+    //    } catch (SQLException ex) {
+    //        return false;
+    //    } finally {
+    //        stmt.close();
+    //    }
+    //}
+
+    //public void setNetworkTimeout(Connection connection,
+    //        Executor executor, int milliseconds) throws SQLException {
+    //    Throwable exception = null;
+    //    if (hasSetNetworkTimeout) {
+    //        try {
+    //            setNetworkTimeoutMethod.invoke(connection, executor, milliseconds);
+    //            return;
+    //
+    //        } catch (IllegalArgumentException ex) {
+    //            // ignore error
+    //            LOG.warn("Connection.setNetworkTimeout failed due to IllegalArgumentException.");
+    //            exception = ex;
+    //
+    //        } catch (IllegalAccessException ex) {
+    //            // ignore error
+    //            LOG.warn("Connection.setNetworkTimeout failed due to IllegalAccessException.");
+    //            exception = ex;
+    //
+    //        } catch (InvocationTargetException ex) {
+    //            //Throwable cause = ex.getTargetException();
+    //            //if (cause instanceof SQLException) {
+    //            //    throw (SQLException) cause;
+    //            //} else if (cause instanceof RuntimeException) {
+    //            //    throw (RuntimeException) cause;
+    //            //} else if (cause instanceof Error) {
+    //            //    throw (Error) cause;
+    //            //} else {
+    //            //    throw new SQLException(cause);
+    //            //}
+    //            exception = ex.getTargetException();
+    //            // It's safe to ignore exceptions.
+    //        }
+    //
+    //        hasSetNetworkTimeout = false;
+    //    }
+    //
+    //    if (exception != null) {
+    //        LOG.warn("Connection.setNetworkTimeout is not available: "+exception);
+    //    } else {
+    //        LOG.warn("Connection.setNetworkTimeout is not available.");
+    //    }
+    //    // TODO any substitute implementations?
+    //}
+}
+

data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java

@@ -1,5 +1,6 @@
 package org.embulk.output.jdbc;
 
+import java.util.List;
 import java.io.IOException;
 import java.math.BigDecimal;
 import java.sql.PreparedStatement;
@@ -7,6 +8,7 @@ import java.sql.SQLException;
 import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
+import com.google.common.base.Optional;
 import org.slf4j.Logger;
 import org.embulk.spi.Exec;
 
@@ -16,6 +18,7 @@ public class StandardBatchInsert
     private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
 
     private final JdbcOutputConnector connector;
+    private final Optional<List<String>> mergeKeys;
 
     private JdbcOutputConnection connection;
     private PreparedStatement batch;
@@ -24,25 +27,25 @@ public class StandardBatchInsert
     private int batchRows;
     private long totalRows;
 
-    public StandardBatchInsert(JdbcOutputConnector connector) throws IOException, SQLException
+    public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
     {
         this.connector = connector;
+        this.mergeKeys = mergeKeys;
     }
 
     public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
     {
         this.connection = connector.connect(true);
-        this.batch = newPreparedStatement(connection, loadTable, insertSchema);
         this.index = 1;  // PreparedStatement index begings from 1
         this.batchRows = 0;
         this.totalRows = 0;
+        this.batch = prepareStatement(loadTable, insertSchema);
         batch.clearBatch();
     }
 
-    protected PreparedStatement newPreparedStatement(JdbcOutputConnection connection,
-                                                     String loadTable, JdbcSchema insertSchema) throws SQLException
+    protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
     {
-        return connection.prepareInsertSql(loadTable, insertSchema);
+        return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
     }
 
     public int getBatchWeight()