embulk-output-jdbc 0.2.4 → 0.3.0

data/src/main/java/org/embulk/output/jdbc/BatchInsert.java

@@ -1,54 +1,54 @@
-package org.embulk.output.jdbc;
-
-import java.math.BigDecimal;
-import java.io.IOException;
-import java.sql.SQLException;
-import java.sql.PreparedStatement;
-import java.sql.Date;
-import java.sql.Time;
-import java.sql.Timestamp;
-
-public interface BatchInsert
-{
-    public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException;
-
-    public int getBatchWeight();
-
-    public void add() throws IOException, SQLException;
-
-    public void close() throws IOException, SQLException;
-
-    public void flush() throws IOException, SQLException;
-
-    public void finish() throws IOException, SQLException;
-
-    public void setNull(int sqlType) throws IOException, SQLException;
-
-    public void setBoolean(boolean v) throws IOException, SQLException;
-
-    public void setByte(byte v) throws IOException, SQLException;
-
-    public void setShort(short v) throws IOException, SQLException;
-
-    public void setInt(int v) throws IOException, SQLException;
-
-    public void setLong(long v) throws IOException, SQLException;
-
-    public void setFloat(float v) throws IOException, SQLException;
-
-    public void setDouble(double v) throws IOException, SQLException;
-
-    public void setBigDecimal(BigDecimal v) throws IOException, SQLException;
-
-    public void setString(String v) throws IOException, SQLException;
-
-    public void setNString(String v) throws IOException, SQLException;
-
-    public void setBytes(byte[] v) throws IOException, SQLException;
-
-    public void setSqlDate(Date v, int sqlType) throws IOException, SQLException;
-
-    public void setSqlTime(Time v, int sqlType) throws IOException, SQLException;
-
-    public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException, SQLException;
-}
+package org.embulk.output.jdbc;
+
+import java.math.BigDecimal;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.sql.PreparedStatement;
+import java.sql.Date;
+import java.sql.Time;
+import java.sql.Timestamp;
+
+public interface BatchInsert
+{
+    public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException;
+
+    public int getBatchWeight();
+
+    public void add() throws IOException, SQLException;
+
+    public void close() throws IOException, SQLException;
+
+    public void flush() throws IOException, SQLException;
+
+    public void finish() throws IOException, SQLException;
+
+    public void setNull(int sqlType) throws IOException, SQLException;
+
+    public void setBoolean(boolean v) throws IOException, SQLException;
+
+    public void setByte(byte v) throws IOException, SQLException;
+
+    public void setShort(short v) throws IOException, SQLException;
+
+    public void setInt(int v) throws IOException, SQLException;
+
+    public void setLong(long v) throws IOException, SQLException;
+
+    public void setFloat(float v) throws IOException, SQLException;
+
+    public void setDouble(double v) throws IOException, SQLException;
+
+    public void setBigDecimal(BigDecimal v) throws IOException, SQLException;
+
+    public void setString(String v) throws IOException, SQLException;
+
+    public void setNString(String v) throws IOException, SQLException;
+
+    public void setBytes(byte[] v) throws IOException, SQLException;
+
+    public void setSqlDate(Date v, int sqlType) throws IOException, SQLException;
+
+    public void setSqlTime(Time v, int sqlType) throws IOException, SQLException;
+
+    public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException, SQLException;
+}

data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java

@@ -1,39 +1,63 @@
 package org.embulk.output.jdbc;
 
+import com.google.common.base.Optional;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 public class JdbcColumn
 {
-    private String name;
-    private String typeName;
-    private int sqlType;
-    private int sizeTypeParameter;
-    private int scaleTypeParameter;
-    private boolean isPrimaryKey;
+    private final String name;
+    private final String simpleTypeName;
+    private final int sqlType;
+    private final int sizeTypeParameter;
+    private final int scaleTypeParameter;
+    private final Optional<String> declaredType;
+    private final boolean isNotNull;
+    private final boolean isUniqueKey;
 
     @JsonCreator
     public JdbcColumn(
             @JsonProperty("name") String name,
-            @JsonProperty("typeName") String typeName,
             @JsonProperty("sqlType") int sqlType,
+            @JsonProperty("simpleTypeName") String simpleTypeName,
             @JsonProperty("sizeTypeParameter") int sizeTypeParameter,
             @JsonProperty("scaleTypeParameter") int scaleTypeParameter,
-            @JsonProperty("primaryKey") boolean isPrimaryKey)
+            @JsonProperty("declaredType") Optional<String> declaredType,
+            @JsonProperty("notNull") boolean isNotNull,
+            @JsonProperty("uniqueKey") boolean isUniqueKey)
     {
         this.name = name;
-        this.typeName = typeName;
+        this.simpleTypeName = simpleTypeName;
         this.sqlType = sqlType;
         this.sizeTypeParameter = sizeTypeParameter;
         this.scaleTypeParameter = scaleTypeParameter;
-        this.isPrimaryKey = isPrimaryKey;
+        this.declaredType = declaredType;
+        this.isNotNull = isNotNull;
+        this.isUniqueKey = isUniqueKey;
+    }
+
+    public static JdbcColumn newGenericTypeColumn(String name, int sqlType,
+            String simpleTypeName, int sizeTypeParameter, int scaleTypeParameter,
+            boolean isNotNull, boolean isUniqueKey)
+    {
+        return new JdbcColumn(name, sqlType,
+                simpleTypeName, sizeTypeParameter, scaleTypeParameter, Optional.<String>absent(),
+                isNotNull, isUniqueKey);
+    }
+
+    public static JdbcColumn newTypeDeclaredColumn(String name, int sqlType,
+            String declaredType, boolean isNotNull, boolean isUniqueKey)
+    {
+        return new JdbcColumn(name, sqlType,
+                declaredType, 0, 0, Optional.of(declaredType),
+                isNotNull, isUniqueKey);
     }
 
     @JsonIgnore
     public static JdbcColumn skipColumn()
     {
-        return new JdbcColumn(null, null, 0, 0, 0, false);
+        return new JdbcColumn(null, 0, null, 0, 0, Optional.<String>absent(), false, false);
     }
 
     @JsonIgnore
@@ -42,30 +66,24 @@ public class JdbcColumn
         return name == null;
     }
 
-    @JsonProperty("primaryKey")
-    public boolean isPrimaryKey()
-    {
-        return isPrimaryKey;
-    }
-
     @JsonProperty("name")
     public String getName()
     {
         return name;
     }
 
-    @JsonProperty("typeName")
-    public String getTypeName()
-    {
-        return typeName;
-    }
-
     @JsonProperty("sqlType")
     public int getSqlType()
     {
         return sqlType;
     }
 
+    @JsonProperty("simpleTypeName")
+    public String getSimpleTypeName()
+    {
+        return simpleTypeName;
+    }
+
     @JsonProperty("sizeTypeParameter")
     public int getSizeTypeParameter()
     {
@@ -77,4 +95,22 @@ public class JdbcColumn
     {
         return scaleTypeParameter;
     }
+
+    @JsonProperty("declaredType")
+    public Optional<String> getDeclaredType()
+    {
+        return declaredType;
+    }
+
+    @JsonProperty("notNull")
+    public boolean isNotNull()
+    {
+        return isNotNull;
+    }
+
+    @JsonProperty("uniqueKey")
+    public boolean isUniqueKey()
+    {
+        return isUniqueKey;
+    }
 }

data/src/main/java/org/embulk/output/jdbc/JdbcColumnOption.java

@@ -0,0 +1,34 @@
+package org.embulk.output.jdbc;
+
+import com.google.common.base.Optional;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.spi.time.TimestampFormat;
+
+public interface JdbcColumnOption
+        extends Task
+{
+    @Config("type")
+    @ConfigDefault("null")
+    public Optional<String> getType();
+
+    @Config("value_type")
+    @ConfigDefault("\"coalesce\"")
+    public String getValueType();
+
+    @Config("timestamp_format")
+    @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"")
+    public TimestampFormat getTimestampFormat();
+
+    @Config("timezone")
+    @ConfigDefault("null")
+    public Optional<DateTimeZone> getTimeZone();
+
+    // required by TimestampFormatter
+    @ConfigInject
+    public ScriptingContainer getJRuby();
+}

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -1,13 +1,16 @@
 package org.embulk.output.jdbc;
 
+import java.util.List;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
-
 import org.slf4j.Logger;
+import com.google.common.base.Optional;
 import org.embulk.spi.Exec;
 
 public class JdbcOutputConnection
@@ -47,6 +50,11 @@ public class JdbcOutputConnection
         return databaseMetaData;
     }
 
+    public Charset getTableNameCharset() throws SQLException
+    {
+        return StandardCharsets.UTF_8;
+    }
+
     protected void setSearchPath(String schema) throws SQLException
     {
         Statement stmt = connection.createStatement();
@@ -124,46 +132,20 @@ public class JdbcOutputConnection
 
         sb.append("CREATE TABLE IF NOT EXISTS ");
         quoteIdentifierString(sb, name);
-        sb.append(buildColumnsOfCreateTableSql(schema));
+        sb.append(buildCreateTableSchemaSql(schema));
         return sb.toString();
     }
 
-    public void createTable(String tableName, JdbcSchema schema) throws SQLException
-    {
-        Statement stmt = connection.createStatement();
-        try {
-            String sql = buildCreateTableSql(tableName, schema);
-            executeUpdate(stmt, sql);
-            commitIfNecessary(connection);
-        } catch (SQLException ex) {
-            throw safeRollback(connection, ex);
-        } finally {
-            stmt.close();
-        }
-    }
-
-    protected String buildCreateTableSql(String name, JdbcSchema schema)
-    {
-        StringBuilder sb = new StringBuilder();
-
-        sb.append("CREATE TABLE ");
-        quoteIdentifierString(sb, name);
-        sb.append(buildColumnsOfCreateTableSql(schema));
-        return sb.toString();
-    }
-
-    private String buildColumnsOfCreateTableSql(JdbcSchema schema)
+    protected String buildCreateTableSchemaSql(JdbcSchema schema)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append(" (");
-        boolean first = true;
-        for (JdbcColumn c : schema.getColumns()) {
-            if (first) { first = false; }
-            else { sb.append(", "); }
-            quoteIdentifierString(sb, c.getName());
+        for (int i=0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, schema.getColumnName(i));
             sb.append(" ");
-            String typeName = getCreateTableTypeName(c);
+            String typeName = getCreateTableTypeName(schema.getColumn(i));
             sb.append(typeName);
         }
         sb.append(")");
@@ -181,33 +163,36 @@ public class JdbcOutputConnection
 
     protected String getCreateTableTypeName(JdbcColumn c)
     {
-        String convertedTypeName = convertTypeName(c.getTypeName());
-        switch (getColumnDeclareType(convertedTypeName, c)) {
+        if (c.getDeclaredType().isPresent()) {
+            return c.getDeclaredType().get();
+        } else {
+            return buildColumnTypeName(c);
+        }
+    }
+
+    protected String buildColumnTypeName(JdbcColumn c)
+    {
+        String simpleTypeName = c.getSimpleTypeName();
+        switch (getColumnDeclareType(simpleTypeName, c)) {
         case SIZE:
-            return String.format("%s(%d)", convertedTypeName, c.getSizeTypeParameter());
+            return String.format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
         case SIZE_AND_SCALE:
             if (c.getScaleTypeParameter() < 0) {
-                return String.format("%s(%d,0)", convertedTypeName, c.getSizeTypeParameter());
+                return String.format("%s(%d,0)", simpleTypeName, c.getSizeTypeParameter());
             } else {
-                return String.format("%s(%d,%d)", convertedTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
+                return String.format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
             }
         case SIZE_AND_OPTIONAL_SCALE:
             if (c.getScaleTypeParameter() < 0) {
-                return String.format("%s(%d)", convertedTypeName, c.getSizeTypeParameter());
+                return String.format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
             } else {
-                return String.format("%s(%d,%d)", convertedTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
+                return String.format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
             }
         default:  // SIMPLE
-            return convertedTypeName;
+            return simpleTypeName;
         }
     }
 
-    // hook point for subclasses
-    protected String convertTypeName(String typeName)
-    {
-        return typeName;
-    }
-
     // TODO
     private static final String[] STANDARD_SIZE_TYPE_NAMES = new String[] {
         "CHAR",
@@ -242,122 +227,118 @@ public class JdbcOutputConnection
         return ColumnDeclareType.SIMPLE;
     }
 
-    protected String buildInsertTableSql(String fromTable, JdbcSchema fromTableSchema, String toTable)
+    public PreparedStatement prepareBatchInsertStatement(String toTable, JdbcSchema toTableSchema, Optional<List<String>> mergeKeys) throws SQLException
+    {
+        String sql;
+        if (mergeKeys.isPresent()) {
+            sql = buildPreparedMergeSql(toTable, toTableSchema, mergeKeys.get());
+        } else {
+            sql = buildPreparedInsertSql(toTable, toTableSchema);
+        }
+        logger.info("Prepared SQL: {}", sql);
+        return connection.prepareStatement(sql);
+    }
+
+    protected String buildPreparedInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
         quoteIdentifierString(sb, toTable);
+
         sb.append(" (");
-        boolean first = true;
-        for (JdbcColumn c : fromTableSchema.getColumns()) {
-            if (first) { first = false; }
-            else { sb.append(", "); }
-            quoteIdentifierString(sb, c.getName());
+        for (int i=0; i < toTableSchema.getCount(); i++) {
+            if(i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, toTableSchema.getColumnName(i));
         }
-        sb.append(") ");
-        sb.append("SELECT ");
-        for (JdbcColumn c : fromTableSchema.getColumns()) {
-            if (first) { first = false; }
-            else { sb.append(", "); }
-            quoteIdentifierString(sb, c.getName());
+        sb.append(") VALUES (");
+        for(int i=0; i < toTableSchema.getCount(); i++) {
+            if(i != 0) { sb.append(", "); }
+            sb.append("?");
         }
-        sb.append(" FROM ");
-        quoteIdentifierString(sb, fromTable);
+        sb.append(")");
 
         return sb.toString();
     }
 
-    protected String buildTruncateSql(String table)
+    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, List<String> mergeKeys) throws SQLException
     {
-        StringBuilder sb = new StringBuilder();
-
-        sb.append("DELETE FROM ");
-        quoteIdentifierString(sb, table);
-
-        return sb.toString();
+        throw new UnsupportedOperationException("not implemented");
     }
 
-    protected void insertTable(String fromTable, JdbcSchema fromTableSchema, String toTable,
+    protected void collectInsert(List<String> fromTables, JdbcSchema schema, String toTable,
             boolean truncateDestinationFirst) throws SQLException
     {
         Statement stmt = connection.createStatement();
         try {
-            if(truncateDestinationFirst) {
+            if (truncateDestinationFirst) {
                 String sql = buildTruncateSql(toTable);
                 executeUpdate(stmt, sql);
             }
-            String sql = buildInsertTableSql(fromTable, fromTableSchema, toTable);
+            String sql = buildCollectInsertSql(fromTables, schema, toTable);
             executeUpdate(stmt, sql);
             commitIfNecessary(connection);
         } catch (SQLException ex) {
-            connection.rollback();
-            throw ex;
+            throw safeRollback(connection, ex);
         } finally {
             stmt.close();
         }
     }
 
-    public PreparedStatement prepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
+    protected String buildTruncateSql(String table)
     {
-        String insertSql = buildPrepareInsertSql(toTable, toTableSchema);
-        logger.info("Prepared SQL: {}", insertSql);
-        return connection.prepareStatement(insertSql);
-    }
+        StringBuilder sb = new StringBuilder();
 
-    public PreparedStatement prepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
-    {
-        String upsertSql = buildPrepareUpsertSql(toTable, toTableSchema);
-        logger.info("Prepared SQL: {}", upsertSql);
-        return connection.prepareStatement(upsertSql);
+        sb.append("DELETE FROM ");
+        quoteIdentifierString(sb, table);
+
+        return sb.toString();
     }
 
-    protected String buildPrepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
+    protected String buildCollectInsertSql(List<String> fromTables, JdbcSchema schema, String toTable)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
         quoteIdentifierString(sb, toTable);
-
         sb.append(" (");
-        for (int i=0; i < toTableSchema.getCount(); i++) {
-            if(i != 0) { sb.append(", "); }
-            quoteIdentifierString(sb, toTableSchema.getColumnName(i));
+        for (int i=0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, schema.getColumnName(i));
         }
-        sb.append(") VALUES (");
-        for(int i=0; i < toTableSchema.getCount(); i++) {
-            if(i != 0) { sb.append(", "); }
-            sb.append("?");
+        sb.append(") ");
+        for (int i=0; i < fromTables.size(); i++) {
+            if (i != 0) { sb.append(" UNION ALL "); }
+            sb.append("SELECT ");
+            for (int j=0; j < schema.getCount(); j++) {
+                if (j != 0) { sb.append(", "); }
+                quoteIdentifierString(sb, schema.getColumnName(j));
+            }
+            sb.append(" FROM ");
+            quoteIdentifierString(sb, fromTables.get(i));
         }
-        sb.append(")");
 
         return sb.toString();
     }
 
-    protected String buildPrepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
+    protected void collectMerge(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
     {
-        throw new UnsupportedOperationException("not implemented yet");
+        Statement stmt = connection.createStatement();
+        try {
+            String sql = buildCollectMergeSql(fromTables, schema, toTable, mergeKeys);
+            executeUpdate(stmt, sql);
+            commitIfNecessary(connection);
+        } catch (SQLException ex) {
+            throw safeRollback(connection, ex);
+        } finally {
+            stmt.close();
+        }
     }
 
-    // TODO
-    //protected void gatherInsertTables(List<String> fromTables, JdbcSchema fromTableSchema, String toTable,
-    //        boolean truncateDestinationFirst) throws SQLException
-    //{
-    //    Statement stmt = connection.createStatement();
-    //    try {
-    //        if(truncateDestinationFirst) {
-    //            String sql = buildTruncateSql(toTable);
-    //            executeUpdate(stmt, sql);
-    //        }
-    //        String sql = buildGatherInsertTables(fromTable, fromTableSchema, toTable);
-    //        executeUpdate(stmt, sql);
-    //        commitIfNecessary(connection);
-    //    } catch (SQLException ex) {
-    //        throw safeRollback(connection, ex);
-    //    } finally {
-    //        stmt.close();
-    //    }
-    //}
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    {
+        throw new UnsupportedOperationException("not implemented");
+    }
 
     public void replaceTable(String fromTable, JdbcSchema schema, String toTable) throws SQLException
     {