embulk-output-jdbc 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3891c0b994ce34234e7b90ded03dacf56fc9f66
-  data.tar.gz: 3cda1683bbdad0cab85467765be2a0c16c03b31d
+  metadata.gz: 1c01d9d1c8f2babb4d908929c0118a5a9c28d87f
+  data.tar.gz: c16e74184b0f5a59b8cac366bcfca99e5a38cb1c
 SHA512:
-  metadata.gz: e6f380f77d762d8cf2f26a370ab883637a1179831d099b492374b297a0e95f52947faf0ce010fdf6073f9a2b71eefee73f3b7153ed6e4a23de04dad67d2dd0b8
-  data.tar.gz: b5f508e16456ad41f61c192d6ee0a5790685c116c2a4803cea7abb8a575981113fb8341af47906fe49d0044e9a300dcfe9aab8f9297eb29b8abe7d61ad20c447
+  metadata.gz: 5a3b33623d041ced79d8c4cb885ddbe8505da46395cb1953523874e58b4eb8a630eed4e9ca8a6c16eb4051bc44dd2c973dbb6cad3f6023daebddf8892eabaffa
+  data.tar.gz: 3df9712d68feca663e6bcbf053c8856ee2dc8779a80515da63457040a7bf0261d1a018e50eac461d10dd0fe4ba408c8e20bd58a4bfefbc6d076f7926f187d79f

data/README.md

@@ -19,8 +19,9 @@ Generic JDBC output plugin for Embulk loads records to a database using a JDBC d
 - **table**: destination table name (string, required)
 - **create_table_constraint**: table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **create_table_option**: table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **transaction_isolation**: transaction isolation level for each connection ("read_uncommitted", "read_committed", "repeatable_read" or "serializable"). if not specified, database default value will be used.
 - **options**: extra JDBC properties (hash, default: {})
-- **retry_limit**: max retry count for database operations (integer, default: 12)
+- **retry_limit**: max retry count for database operations (integer, default: 12). When intermediate table to create already created by another process, this plugin will retry with another table name to avoid collision.
 - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See below (string, required)

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputConnector.java

@@ -0,0 +1,24 @@
+package org.embulk.output.jdbc;
+
+import java.sql.SQLException;
+
+import com.google.common.base.Optional;
+
+public abstract class AbstractJdbcOutputConnector implements JdbcOutputConnector
+{
+    private final Optional<TransactionIsolation> transactionIsolation;
+
+    public AbstractJdbcOutputConnector(Optional<TransactionIsolation> transactionIsolation)
+    {
+        this.transactionIsolation = transactionIsolation;
+    }
+
+    public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
+    {
+        JdbcOutputConnection connection = connect();
+        connection.initialize(autoCommit, transactionIsolation);
+        return connection;
+    }
+
+    protected abstract JdbcOutputConnection connect() throws SQLException;
+}

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -128,6 +128,11 @@ public abstract class AbstractJdbcOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getAfterLoad();
 
+        @Config("transaction_isolation")
+        @ConfigDefault("null")
+        public Optional<TransactionIsolation> getTransactionIsolation();
+        public void setTransactionIsolation(Optional<TransactionIsolation> transactionIsolation);
+
         public void setActualTable(TableIdentifier actualTable);
         public TableIdentifier getActualTable();
 
@@ -1070,25 +1075,27 @@ public abstract class AbstractJdbcOutputPlugin
             implements TransactionalPageOutput
     {
         protected final List<Column> columns;
+        protected final List<ColumnSetter> columnSetters;
         protected final List<ColumnSetterVisitor> columnVisitors;
-        private final PageReader pageReader;
+        private final PageReaderRecord pageReader;
         private final BatchInsert batch;
         private final int batchSize;
         private final int forceBatchFlushSize;
         private final PluginTask task;
 
-        public PluginPageOutput(final PageReader pageReader,
+        public PluginPageOutput(PageReader pageReader,
                 BatchInsert batch, List<ColumnSetter> columnSetters,
                 int batchSize, PluginTask task)
         {
-            this.pageReader = pageReader;
+            this.pageReader = new PageReaderRecord(pageReader);
             this.batch = batch;
             this.columns = pageReader.getSchema().getColumns();
+            this.columnSetters = columnSetters;
             this.columnVisitors = ImmutableList.copyOf(Lists.transform(
                         columnSetters, new Function<ColumnSetter, ColumnSetterVisitor>() {
                             public ColumnSetterVisitor apply(ColumnSetter setter)
                             {
-                                return new ColumnSetterVisitor(pageReader, setter);
+                                return new ColumnSetterVisitor(PluginPageOutput.this.pageReader, setter);
                             }
                         }));
             this.batchSize = batchSize;
@@ -1103,49 +1110,57 @@ public abstract class AbstractJdbcOutputPlugin
                 pageReader.setPage(page);
                 while (pageReader.nextRecord()) {
                     if (batch.getBatchWeight() > forceBatchFlushSize) {
-                        withRetry(task, new IdempotentSqlRunnable() {
-                            @Override
-                            public void run() throws SQLException {
-                                try {
-                                    batch.flush();
-                                } catch (IOException ex) {
-                                    throw new RuntimeException(ex);
-                                }
-                            }
-                        });
+                        flush();
                     }
                     handleColumnsSetters();
                     batch.add();
                 }
                 if (batch.getBatchWeight() > batchSize) {
-                    withRetry(task, new IdempotentSqlRunnable() {
-                        @Override
-                        public void run() throws SQLException {
-                            try {
-                                batch.flush();
-                            } catch (IOException ex) {
-                                throw new RuntimeException(ex);
-                            }
-                        }
-                    });
+                    flush();
                 }
             } catch (IOException | SQLException | InterruptedException ex) {
                 throw new RuntimeException(ex);
             }
         }
 
+        private void flush() throws SQLException, InterruptedException
+        {
+            withRetry(task, new IdempotentSqlRunnable() {
+                private boolean first = true;
+
+                @Override
+                public void run() throws IOException, SQLException {
+                    try {
+                        if (!first) {
+                            retryColumnsSetters();
+                        }
+
+                        batch.flush();
+
+                    } catch (IOException | SQLException ex) {
+                        if (!first && !isRetryableException(ex)) {
+                            logger.error("Retry failed : ", ex);
+                        }
+                        throw ex;
+                    } finally {
+                        first = false;
+                    }
+                }
+            });
+
+            pageReader.clearReadRecords();
+        }
+
         @Override
         public void finish()
         {
             try {
+                flush();
+
                 withRetry(task, new IdempotentSqlRunnable() {
                     @Override
-                    public void run() throws SQLException {
-                        try {
-                            batch.finish();
-                        } catch (IOException ex) {
-                            throw new RuntimeException(ex);
-                        }
+                    public void run() throws IOException, SQLException {
+                        batch.finish();
                     }
                 });
             } catch (InterruptedException | SQLException ex) {
@@ -1181,12 +1196,30 @@ public abstract class AbstractJdbcOutputPlugin
                 columns.get(i).visit(columnVisitors.get(i));
             }
         }
+
+        protected void retryColumnsSetters() throws IOException, SQLException
+        {
+            int size = columnVisitors.size();
+            for (Record record : pageReader.getReadRecords()) {
+                for (int i = 0; i < size; i++) {
+                    ColumnSetterVisitor columnVisitor = new ColumnSetterVisitor(record, columnSetters.get(i));
+                    columns.get(i).visit(columnVisitor);
+                }
+                batch.add();
+            }
+        }
     }
 
-    protected boolean isRetryableException(SQLException exception)
+    protected boolean isRetryableException(Exception exception)
     {
-        return isRetryableException(exception.getSQLState(), exception.getErrorCode());
+        if (exception instanceof SQLException) {
+            SQLException ex = (SQLException)exception;
+            return isRetryableException(ex.getSQLState(), ex.getErrorCode());
+        } else {
+            return false;
+        }
     }
+
     protected boolean isRetryableException(String sqlState, int errorCode)
     {
         return false;
@@ -1195,7 +1228,7 @@ public abstract class AbstractJdbcOutputPlugin
 
     public static interface IdempotentSqlRunnable
     {
-        public void run() throws SQLException;
+        public void run() throws IOException, SQLException;
     }
 
     protected void withRetry(PluginTask task, IdempotentSqlRunnable op)
@@ -1269,17 +1302,13 @@ public abstract class AbstractJdbcOutputPlugin
             }
         }
 
-        public boolean isRetryableException(Exception exception) {
-            if (exception instanceof SQLException) {
-                SQLException ex = (SQLException)exception;
-
-                return AbstractJdbcOutputPlugin.this.isRetryableException(ex);
-            } else {
-                return false;
-            }
+        public boolean isRetryableException(Exception exception)
+        {
+            return AbstractJdbcOutputPlugin.this.isRetryableException(exception);
         }
 
-        private String buildExceptionMessage(Throwable ex) {
+        private String buildExceptionMessage(Throwable ex)
+        {
             StringBuilder sb = new StringBuilder();
             sb.append(ex.getMessage());
             if (ex.getCause() != null) {
@@ -1288,7 +1317,8 @@ public abstract class AbstractJdbcOutputPlugin
             return sb.toString();
         }
 
-        private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
+        private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage)
+        {
             if (!lastMessage.equals(ex.getMessage())) {
                 // suppress same messages
                 sb.append(" < ");

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -38,6 +38,23 @@ public class JdbcOutputConnection
         }
     }
 
+    public void initialize(boolean autoCommit, Optional<TransactionIsolation> transactionIsolation)
+            throws SQLException
+    {
+        connection.setAutoCommit(autoCommit);
+
+        if (transactionIsolation.isPresent()) {
+            connection.setTransactionIsolation(transactionIsolation.get().toInt());
+        }
+
+        try {
+            TransactionIsolation currentTransactionIsolation = TransactionIsolation.fromInt(connection.getTransactionIsolation());
+            logger.info("TransactionIsolation={}", currentTransactionIsolation.toString());
+        } catch (IllegalArgumentException e) {
+            logger.info("TransactionIsolation=unknown");
+        }
+    }
+
     @Override
     public void close() throws SQLException
     {

data/src/main/java/org/embulk/output/jdbc/MemoryRecord.java

@@ -0,0 +1,63 @@
+package org.embulk.output.jdbc;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+
+public class MemoryRecord implements Record
+{
+    private final Object[] values;
+
+    public MemoryRecord(int columnCount)
+    {
+        values = new Object[columnCount];
+    }
+
+
+    public boolean isNull(Column column)
+    {
+        return getValue(column) == null;
+    }
+
+    public boolean getBoolean(Column column)
+    {
+        return (Boolean)getValue(column);
+    }
+
+    public long getLong(Column column)
+    {
+        return (Long)getValue(column);
+    }
+
+    public double getDouble(Column column)
+    {
+        return (Double)getValue(column);
+    }
+
+    public String getString(Column column)
+    {
+        return (String)getValue(column);
+    }
+
+    public Timestamp getTimestamp(Column column)
+    {
+        return (Timestamp)getValue(column);
+    }
+
+    public Value getJson(Column column)
+    {
+        return (Value)getValue(column);
+    }
+
+    private Object getValue(Column column)
+    {
+        return values[column.getIndex()];
+    }
+
+    public void setValue(Column column, Object value)
+    {
+        values[column.getIndex()] = value;
+    }
+}
+

data/src/main/java/org/embulk/output/jdbc/PageReaderRecord.java

@@ -0,0 +1,96 @@
+package org.embulk.output.jdbc;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+
+/**
+ * Record read by PageReader.
+ * The class will save read records for retry.
+ */
+public class PageReaderRecord implements Record
+{
+    private final PageReader pageReader;
+    private final List<MemoryRecord> readRecords;
+    private MemoryRecord lastRecord;
+
+    public PageReaderRecord(PageReader pageReader)
+    {
+        this.pageReader = pageReader;
+        readRecords = new ArrayList<MemoryRecord>();
+    }
+
+    public void setPage(Page page)
+    {
+        pageReader.setPage(page);
+    }
+
+    public boolean nextRecord()
+    {
+        lastRecord = null; // lastRecord will be created in next `save` method execution.
+        return pageReader.nextRecord();
+    }
+
+    public boolean isNull(Column column)
+    {
+        return pageReader.isNull(column);
+    }
+
+    public boolean getBoolean(Column column)
+    {
+        return save(column, pageReader.getBoolean(column));
+    }
+
+    public long getLong(Column column)
+    {
+        return save(column, pageReader.getLong(column));
+    }
+
+    public double getDouble(Column column)
+    {
+        return save(column, pageReader.getDouble(column));
+    }
+
+    public String getString(Column column)
+    {
+        return save(column, pageReader.getString(column));
+    }
+
+    public Timestamp getTimestamp(Column column)
+    {
+        return save(column, pageReader.getTimestamp(column));
+    }
+
+    public Value getJson(Column column)
+    {
+        return save(column, pageReader.getJson(column));
+    }
+
+    public List<? extends Record> getReadRecords()
+    {
+        return readRecords;
+    }
+
+    public void clearReadRecords()
+    {
+        readRecords.clear();
+        lastRecord = null;
+    }
+
+    private <T> T save(Column column, T value)
+    {
+        if (lastRecord == null) {
+            lastRecord = new MemoryRecord(pageReader.getSchema().getColumnCount());
+            readRecords.add(lastRecord);
+        }
+        lastRecord.setValue(column, value);
+        return value;
+    }
+}
+

data/src/main/java/org/embulk/output/jdbc/Record.java

@@ -0,0 +1,23 @@
+package org.embulk.output.jdbc;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public interface Record
+{
+    boolean isNull(Column column);
+
+    boolean getBoolean(Column column);
+
+    long getLong(Column column);
+
+    double getDouble(Column column);
+
+    String getString(Column column);
+
+    Timestamp getTimestamp(Column column);
+
+    Value getJson(Column column);
+}
+

data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java

@@ -70,23 +70,27 @@ public class StandardBatchInsert
 
     public void flush() throws IOException, SQLException
     {
+        if (batchWeight == 0) return;
+
         logger.info(String.format("Loading %,d rows", batchRows));
         long startTime = System.currentTimeMillis();
-        batch.executeBatch();  // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
-        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
-
-        totalRows += batchRows;
-        logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
-        batch.clearBatch();
-        batchRows = 0;
-        batchWeight = 0;
+        try {
+            batch.executeBatch();  // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
+            double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+
+            totalRows += batchRows;
+            logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
+
+        } finally {
+            // clear for retry
+            batch.clearBatch();
+            batchRows = 0;
+            batchWeight = 0;
+        }
     }
 
     public void finish() throws IOException, SQLException
     {
-        if (getBatchWeight() != 0) {
-            flush();
-        }
     }
 
     public void setNull(int sqlType) throws IOException, SQLException

data/src/main/java/org/embulk/output/jdbc/TransactionIsolation.java

@@ -0,0 +1,66 @@
+package org.embulk.output.jdbc;
+
+import java.sql.Connection;
+import java.util.Locale;
+
+import org.embulk.config.ConfigException;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public enum TransactionIsolation {
+    READ_UNCOMMITTED {
+        @Override
+        public int toInt() {
+            return Connection.TRANSACTION_READ_UNCOMMITTED;
+        }
+    },
+    READ_COMMITTED {
+        @Override
+        public int toInt() {
+            return Connection.TRANSACTION_READ_COMMITTED;
+        }
+    },
+    REPEATABLE_READ {
+        @Override
+        public int toInt() {
+            return Connection.TRANSACTION_REPEATABLE_READ;
+        }
+    },
+    SERIALIZABLE {
+        @Override
+        public int toInt() {
+            return Connection.TRANSACTION_SERIALIZABLE;
+        }
+    };
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+    public abstract int toInt();
+
+    @JsonCreator
+    public static TransactionIsolation fromString(String value)
+    {
+        for (TransactionIsolation ti : values()) {
+            if (ti.toString().equals(value)) {
+                return ti;
+            }
+        }
+        throw new ConfigException(String.format("Unknown transaction_isolation '%s'.", value));
+    }
+
+    public static TransactionIsolation fromInt(int value)
+    {
+        for (TransactionIsolation ti : values()) {
+            if (ti.toInt() == value) {
+                return ti;
+            }
+        }
+        throw new IllegalArgumentException(String.format("Unknown transaction_isolation '%d'.", value));
+    }
+}

data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterVisitor.java

@@ -2,19 +2,20 @@ package org.embulk.output.jdbc.setter;
 
 import java.io.IOException;
 import java.sql.SQLException;
+
+import org.embulk.output.jdbc.Record;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.PageReader;
 
 public class ColumnSetterVisitor
         implements ColumnVisitor
 {
-    private final PageReader pageReader;
+    private final Record record;
     private final ColumnSetter setter;
 
-    public ColumnSetterVisitor(PageReader pageReader, ColumnSetter setter)
+    public ColumnSetterVisitor(Record record, ColumnSetter setter)
     {
-        this.pageReader = pageReader;
+        this.record = record;
         this.setter = setter;
     }
 
@@ -22,10 +23,10 @@ public class ColumnSetterVisitor
     public void booleanColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.booleanValue(pageReader.getBoolean(column));
+                setter.booleanValue(record.getBoolean(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class
@@ -37,10 +38,10 @@ public class ColumnSetterVisitor
     public void longColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.longValue(pageReader.getLong(column));
+                setter.longValue(record.getLong(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class
@@ -52,10 +53,10 @@ public class ColumnSetterVisitor
     public void doubleColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.doubleValue(pageReader.getDouble(column));
+                setter.doubleValue(record.getDouble(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class
@@ -67,10 +68,10 @@ public class ColumnSetterVisitor
     public void stringColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.stringValue(pageReader.getString(column));
+                setter.stringValue(record.getString(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class
@@ -82,10 +83,10 @@ public class ColumnSetterVisitor
     public void jsonColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.jsonValue(pageReader.getJson(column));
+                setter.jsonValue(record.getJson(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class
@@ -97,10 +98,10 @@ public class ColumnSetterVisitor
     public void timestampColumn(Column column)
     {
         try {
-            if (pageReader.isNull(column)) {
+            if (record.isNull(column)) {
                 setter.nullValue();
             } else {
-                setter.timestampValue(pageReader.getTimestamp(column));
+                setter.timestampValue(record.getTimestamp(column));
             }
         } catch (IOException | SQLException ex) {
             // TODO exception class

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-30 00:00:00.000000000 Z
+date: 2019-05-07 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,9 +19,10 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.2.jar
+- classpath/embulk-output-jdbc-0.8.3.jar
 - lib/embulk/output/jdbc.rb
 - src/main/java/org/embulk/output/JdbcOutputPlugin.java
+- src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputConnector.java
 - src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java
 - src/main/java/org/embulk/output/jdbc/BatchInsert.java
 - src/main/java/org/embulk/output/jdbc/JdbcColumn.java
@@ -30,13 +31,17 @@ files:
 - src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java
 - src/main/java/org/embulk/output/jdbc/JdbcSchema.java
 - src/main/java/org/embulk/output/jdbc/JdbcUtils.java
+- src/main/java/org/embulk/output/jdbc/MemoryRecord.java
 - src/main/java/org/embulk/output/jdbc/MergeConfig.java
+- src/main/java/org/embulk/output/jdbc/PageReaderRecord.java
+- src/main/java/org/embulk/output/jdbc/Record.java
 - src/main/java/org/embulk/output/jdbc/Ssl.java
 - src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java
 - src/main/java/org/embulk/output/jdbc/TableIdentifier.java
 - src/main/java/org/embulk/output/jdbc/TimestampFormat.java
 - src/main/java/org/embulk/output/jdbc/ToString.java
 - src/main/java/org/embulk/output/jdbc/ToStringMap.java
+- src/main/java/org/embulk/output/jdbc/TransactionIsolation.java
 - src/main/java/org/embulk/output/jdbc/setter/BigDecimalColumnSetter.java
 - src/main/java/org/embulk/output/jdbc/setter/BooleanColumnSetter.java
 - src/main/java/org/embulk/output/jdbc/setter/ByteColumnSetter.java