embulk-output-teradata 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f842427d2bb1abc2570877c1f67c24d3815667e9
-  data.tar.gz: 4e2029b8b0a6690025c81799cc175b6025f4aadb
+  metadata.gz: 1576a329c01404b00bfd22be516a1cdf3a1a06eb
+  data.tar.gz: cdf50889dd17602ffd2e1bed867277c0fdfbe9f1
 SHA512:
-  metadata.gz: 911f4df3c54ffcb1b9961250cf001a2c7f129bd06f5d24c6060c58367ec9e9c8c3ce36107827cf841c994676d057556feab03110b0b2df41ed8fe1c5dc3a6906
-  data.tar.gz: e197945ceb751127d85af9eaf6cf2ed7a276befdfe1c6db9d40d8b7a0d2998d19eae05505e3014b67033bc259ec6965fbb9646fa91b7da7bf2ee9ac50eaee0a1
+  metadata.gz: 12bc72528547067922f58487932e1a282ccf9f676fe0268519bec8450993ae242ae4c2781ff4ab84127b494117dcfe82365454e10a9b161040a68cd94aa9c3bd
+  data.tar.gz: 2a64ff07020a0c7a0338d8f59ea6b67633712dff22cb5243829605409cdc8d5e8beeb65abfd4eda67f4be70064a95ba96cbb260ee3775848175e289e82e55212

data/README.md

@@ -48,14 +48,6 @@ Teradata output plugin for Embulk loads records to a database using a Teradata J
   * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
   * Transactional: No. If fails, the target table could be dropped.
   * Resumable: No.
-* **merge**:
-  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, merges the intermediate tables into the target table. Namely, if primary keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
-  * Transactional: Yes.
-  * Resumable: Yes.
-* **merge_direct**:
-  * Behavior: This mode merges rows to the target table directly. Namely, if primary keys of an input record already exist in the target table, the target record is updated by the input record, otherwise the input record is inserted. If the target table doesn't exist, it is created automatically.
-  * Transactional: No.
-  * Resumable: No.
 
 ## Example
 

data/build.gradle

@@ -6,14 +6,16 @@ plugins {
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
-    mavenCentral()
+    maven {
+        url("https://plugins.gradle.org/m2/")
+    }
     jcenter()
 }
 configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -21,6 +23,7 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"
     provided "org.embulk:embulk-core:0.8.2"
+    compile  files('lib/embulk-output-jdbc-0.7.11.jar')
     compile  files('lib/tdgssconfig.jar')
     compile  files('lib/terajdbc4.jar')
 

data/embulk-output-teradata.iml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module external.linked.project.id="embulk-output-teradata" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.1.2" type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.gradle" />
+      <excludeFolder url="file://$MODULE_DIR$/build" />
+    </content>
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Tue Aug 29 21:53:01 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-all.zip

data/src/main/java/org/embulk/output/teradata/TeradataOutputPlugin.java

@@ -1,25 +1,26 @@
 package org.embulk.output.teradata;
 
 import java.util.Properties;
-import java.sql.Driver;
 import java.io.IOException;
-import java.sql.Connection;
 import java.sql.SQLException;
 
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableSet;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.output.jdbc.*;
-import org.embulk.output.teradata.TeradataOutputConnection;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.teradata.jdbc.TeradataBatchInsert;
+import org.embulk.output.teradata.jdbc.TeradataOutputConnector;
 
 public class TeradataOutputPlugin
         extends AbstractJdbcOutputPlugin
 {
 
-    public interface GenericPluginTask extends PluginTask
+    public interface TeradataPluginTask
+            extends PluginTask
     {
         @Config("url")
         public String getUrl();
@@ -44,22 +45,22 @@ public class TeradataOutputPlugin
     @Override
     protected Class<? extends PluginTask> getTaskClass()
     {
-        return GenericPluginTask.class;
+        return TeradataPluginTask.class;
     }
 
     @Override
     protected Features getFeatures(PluginTask task)
     {
-        GenericPluginTask t = (GenericPluginTask) task;
+        TeradataPluginTask t = (TeradataPluginTask) task;
         return new Features()
             .setMaxTableNameLength(t.getMaxTableNameLength())
             .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE));
     }
 
     @Override
-    protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    protected TeradataOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
     {
-        GenericPluginTask t = (GenericPluginTask) task;
+        TeradataPluginTask t = (TeradataPluginTask) task;
 
         Properties props = new Properties();
 
@@ -73,52 +74,12 @@ public class TeradataOutputPlugin
             props.setProperty("password", t.getPassword().get());
         }
 
-        return new GenericOutputConnector(t.getUrl(), props, null, null);
-    }
-
-    private static class GenericOutputConnector
-            implements JdbcOutputConnector
-    {
-        private final Driver driver;
-        private final String url;
-        private final Properties properties;
-        private final String schemaName;
-
-        public GenericOutputConnector(String url, Properties properties, String driverClass,
-                String schemaName)
-        {
-            try {
-                // TODO check Class.forName(driverClass) is a Driver before newInstance
-                //      for security
-                this.driver = new com.teradata.jdbc.TeraDriver();
-            } catch (Exception ex) {
-                throw Throwables.propagate(ex);
-            }
-            this.url = url;
-            this.properties = properties;
-            this.schemaName = schemaName;
-        }
-
-        @Override
-        public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
-        {
-            Connection c = driver.connect(url, properties);
-            try {
-                c.setAutoCommit(autoCommit);
-                JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
-                c = null;
-                return con;
-            } finally {
-                if (c != null) {
-                    c.close();
-                }
-            }
-        }
+        return new TeradataOutputConnector(t.getUrl(), props);
     }
 
     @Override
     protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
-        return new StandardBatchInsert(getConnector(task, true), mergeConfig);
+        return new TeradataBatchInsert(getConnector(task, true), mergeConfig);
     }
 }

data/src/main/java/org/embulk/output/teradata/jdbc/TeradataBatchInsert.java

@@ -0,0 +1,37 @@
+package org.embulk.output.teradata.jdbc;
+
+import java.io.IOException;
+import java.sql.Types;
+import java.sql.SQLException;
+import com.google.common.base.Optional;
+import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.StandardBatchInsert;
+
+public class TeradataBatchInsert
+        extends StandardBatchInsert
+{
+    public TeradataBatchInsert(TeradataOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
+    {
+        super(connector, mergeConfig);
+    }
+
+    @Override
+    public void setFloat(float v) throws IOException, SQLException
+    {
+        if (Float.isNaN(v) || Float.isInfinite(v)) {
+            setNull(Types.REAL);  // TODO get through argument
+        } else {
+            super.setFloat(v);
+        }
+    }
+
+    @Override
+    public void setDouble(double v) throws IOException, SQLException
+    {
+        if (Double.isNaN(v) || Double.isInfinite(v)) {
+            setNull(Types.DOUBLE);  // TODO get through argument
+        } else {
+            super.setDouble(v);
+        }
+    }
+}

data/src/main/java/org/embulk/output/teradata/jdbc/TeradataOutputConnection.java

@@ -0,0 +1,307 @@
+package org.embulk.output.teradata.jdbc;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.sql.Connection;
+import java.sql.DatabaseMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.TableIdentifier;
+import org.slf4j.Logger;
+import org.embulk.spi.Exec;
+
+import static java.lang.String.format;
+
+public class TeradataOutputConnection
+        extends JdbcOutputConnection
+{
+    private final Logger logger = Exec.getLogger(TeradataOutputConnection.class);
+    protected final Connection connection;
+    protected final DatabaseMetaData databaseMetaData;
+    protected String identifierQuoteString;
+
+    public TeradataOutputConnection(Connection connection, boolean autoCommit)
+            throws SQLException
+    {
+        super(connection, null);
+        this.connection = connection;
+        this.databaseMetaData = connection.getMetaData();
+        this.identifierQuoteString = databaseMetaData.getIdentifierQuoteString();
+        if (schemaName != null) {
+            setSearchPath(schemaName);
+        }
+    }
+
+    @Override
+    public void close() throws SQLException
+    {
+        if (!connection.isClosed()) {
+            connection.close();
+        }
+    }
+
+    public String getSchemaName()
+    {
+        return schemaName;
+    }
+
+    public DatabaseMetaData getMetaData() throws SQLException
+    {
+        return databaseMetaData;
+    }
+
+    public Charset getTableNameCharset() throws SQLException
+    {
+        return StandardCharsets.UTF_8;
+    }
+
+    protected void setSearchPath(String schema) throws SQLException
+    {
+        Statement stmt = connection.createStatement();
+        try {
+            String sql = "DATABASE " + quoteIdentifierString(schema);
+            executeUpdate(stmt, sql);
+            commitIfNecessary(connection);
+        } finally {
+            stmt.close();
+        }
+    }
+
+    // Teradata doesn't support CREATE TABLE IF NOT EXIST
+    @Override
+    protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("CREATE TABLE ");
+        this.quoteTableIdentifier(sb, table);
+        sb.append(this.buildCreateTableSchemaSql(schema));
+        return sb.toString();
+    }
+
+    @Override
+    protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
+    {
+        if (existTable(table.getTableName()))
+        {
+            String sql = format("DROP TABLE %s", this.quoteTableIdentifier(table));
+            this.executeUpdate(stmt, sql);
+        }
+        else
+        {
+            logger.info(format("Skip dropping the table: %s", quoteIdentifierString(table.getTableName())));
+        }
+    }
+
+    // Teradata doesn't support IF EXISTS
+    public boolean existTable(String tableName)
+    {
+      try{
+          String sql = format("SELECT COUNT(1) FROM %s", quoteIdentifierString(tableName));
+        executeSql(sql);
+      } catch (SQLException se){
+        return false;
+      }
+      return true;
+    }
+
+    protected String buildCreateTableSchemaSql(JdbcSchema schema)
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(" (");
+        for (int i=0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, schema.getColumnName(i));
+            sb.append(" ");
+            String typeName = getCreateTableTypeName(schema.getColumn(i));
+            sb.append(typeName);
+        }
+        sb.append(")");
+
+        return sb.toString();
+    }
+
+    protected String getCreateTableTypeName(JdbcColumn c)
+    {
+        if (c.getDeclaredType().isPresent()) {
+            return c.getDeclaredType().get();
+        } else {
+            return buildColumnTypeName(c);
+        }
+    }
+
+    protected String buildColumnTypeName(JdbcColumn c)
+    {
+        String simpleTypeName = c.getSimpleTypeName();
+        switch (getColumnDeclareType(simpleTypeName, c)) {
+        case SIZE:
+            return format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
+        case SIZE_AND_SCALE:
+            if (c.getScaleTypeParameter() < 0) {
+                return format("%s(%d,0)", simpleTypeName, c.getSizeTypeParameter());
+            } else {
+                return format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
+            }
+        case SIZE_AND_OPTIONAL_SCALE:
+            if (c.getScaleTypeParameter() < 0) {
+                return format("%s(%d)", simpleTypeName, c.getSizeTypeParameter());
+            } else {
+                return format("%s(%d,%d)", simpleTypeName, c.getSizeTypeParameter(), c.getScaleTypeParameter());
+            }
+        default:  // SIMPLE
+            if (simpleTypeName.equals("CLOB"))
+            {
+                return "VARCHAR(1024)";
+            }
+            return simpleTypeName;
+        }
+    }
+
+    protected void executeSql(String sql) throws SQLException
+    {
+        Statement stmt = connection.createStatement();
+        try {
+            executeUpdate(stmt, sql);
+            commitIfNecessary(connection);
+        } catch (SQLException ex) {
+            throw safeRollback(connection, ex);
+        } finally {
+            stmt.close();
+        }
+    }
+
+    @Override
+    protected String buildRenameTableSql(TableIdentifier fromTable, TableIdentifier toTable) {
+        StringBuilder sb = new StringBuilder();
+        sb.append("RENAME TABLE ");
+        this.quoteTableIdentifier(sb, fromTable);
+        sb.append(" TO ");
+        this.quoteTableIdentifier(sb, toTable);
+        return sb.toString();
+    }
+
+
+    protected void quoteIdentifierString(StringBuilder sb, String str)
+    {
+        sb.append(quoteIdentifierString(str, identifierQuoteString));
+    }
+
+    protected String quoteIdentifierString(String str)
+    {
+        return quoteIdentifierString(str, identifierQuoteString);
+    }
+
+    protected String quoteIdentifierString(String str, String quoteString)
+    {
+        // TODO if identifierQuoteString.equals(" ") && str.contains([^a-zA-Z0-9_connection.getMetaData().getExtraNameCharacters()])
+        // TODO if str.contains(identifierQuoteString);
+        return quoteString + str + quoteString;
+    }
+
+    public boolean isValidConnection(int timeout) throws SQLException
+    {
+        Statement stmt = connection.createStatement();
+        try {
+            stmt.executeQuery("SELECT 1").close();
+            return true;
+        } catch (SQLException ex) {
+            return false;
+        } finally {
+            stmt.close();
+        }
+    }
+
+    protected String[] getDeterministicSqlStates()
+    {
+        return new String[0];
+    }
+
+    protected int[] getDeterministicErrorCodes()
+    {
+        return new int[0];
+    }
+
+    protected Class[] getDeterministicRootCauses()
+    {
+        return new Class[] {
+            // Don't retry on UnknownHostException.
+            java.net.UnknownHostException.class,
+
+            //// we should not retry on connect() error?
+            //java.net.ConnectException.class,
+        };
+    }
+
+    public boolean isRetryableException(SQLException exception)
+    {
+        String sqlState = exception.getSQLState();
+        for (String deterministic : getDeterministicSqlStates()) {
+            if (sqlState.equals(deterministic)) {
+                return false;
+            }
+        }
+
+        int errorCode = exception.getErrorCode();
+        for (int deterministic : getDeterministicErrorCodes()) {
+            if (errorCode == deterministic) {
+                return false;
+            }
+        }
+
+        Throwable rootCause = getRootCause(exception);
+        for (Class deterministic : getDeterministicRootCauses()) {
+            if (deterministic.equals(rootCause.getClass())) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    private Throwable getRootCause(Throwable e) {
+        while (e.getCause() != null) {
+            e = e.getCause();
+        }
+        return e;
+    }
+
+    protected int executeUpdate(Statement stmt, String sql) throws SQLException
+    {
+        logger.info("SQL: " + sql);
+        long startTime = System.currentTimeMillis();
+        int count = stmt.executeUpdate(sql);
+        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+        if (count == 0) {
+            logger.info(format("> %.2f seconds", seconds));
+        } else {
+            logger.info(format("> %.2f seconds (%,d rows)", seconds, count));
+        }
+        return count;
+    }
+
+    protected void commitIfNecessary(Connection con) throws SQLException
+    {
+        if (!con.getAutoCommit()) {
+            con.commit();
+        }
+    }
+
+    protected SQLException safeRollback(Connection con, SQLException cause)
+    {
+        try {
+            if (!con.getAutoCommit()) {
+                con.rollback();
+            }
+            return cause;
+        } catch (SQLException ex) {
+            if (cause != null) {
+                cause.addSuppressed(ex);
+                return cause;
+            }
+            return ex;
+        }
+    }
+}