embulk-output-db2 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b20a23693346957b15458772f8cc1559d4177160
+  data.tar.gz: 6476a9a55e25f30e72ce49a12d34aaa3286ce605
+SHA512:
+  metadata.gz: fbbd4afe3678831d8f7cd751c81894519a4d677af7e3e82a89f182533d22e7288aa6bec737e45e7637ef0612a85695a34329be15d3538e5312f95230cb208965
+  data.tar.gz: 8682f185c18d3a5a5e112c368c7883cfc5bf71d020bf4e781faf51451d2e3cb80ff4a40782403867f2c99fc6088b28bc1b7b44b549dc60d204d3e42f4ae6f9fa

data/README.md

@@ -0,0 +1,118 @@
+# DB2 output plugins for Embulk
+
+DB2 output plugins for Embulk loads records to DB2.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depends on the mode. see below.
+* **Resume supported**: depends on the mode. see below.
+
+## Configuration
+
+- **driver_path**: path to the jar file of the DB2 JDBC driver (string)
+- **host**: database host name (string, required)
+- **port**: database port number (integer, default: 50000)
+- **user**: database login user name (string, required)
+- **password**: database login password (string)
+- **database**: destination database name (string, required)
+- **schema**: destination schema name (string, default: use the default schema)
+- **table**: destination table name (string, required)
+- **options**: extra connection properties (hash, default: {})
+- **retry_limit** max retry count for database operations (integer, default: 12)
+- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
+- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
+- **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See below. (string, required)
+- **insert_method**: see below
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, insert_truncate and merge modes), when it creates the target table (insert_direct, merge_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `SMALLINT` if boolean, `DOUBLE` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
+  - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
+  - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+
+### Modes
+
+* **insert**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ...` query. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes. This mode successfully writes all rows, or fails with writing zero rows.
+  * Resumable: Yes.
+* **insert_direct**:
+  * Behavior: This mode inserts rows to the target table directly. If the target table doesn't exist, it is created automatically.
+  * Transactional: No. If fails, the target table could have some rows inserted.
+  * Resumable: No.
+* **truncate_insert**:
+  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
+  * Transactional: Yes.
+  * Resumable: Yes.
+* **replace**:
+  * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
+  * Transactional: No. If fails, the target table could be dropped (because DB2 can't rollback DDL).
+  * Resumable: No.
+
+### Supported types
+
+|database type|default value_type|note|
+|:--|:--|:--|
+|SMALLINT|short||
+|INTEGER|int||
+|BITINT|long||
+|DECIMAL|decimal||
+|NUMERIC|decimal||
+|REAL|float||
+|DOUBLE|double||
+|FLOAT|double||
+|CHAR|string||
+|VARCHAR|string||
+|CLOB|string||
+|GRAPHIC|string||
+|VARGRAPHIC|string||
+|NCHAR|string||
+|NVARCHAR|string||
+|NCLOB|string||
+|DATE|date||
+|TIME|time||
+|TIMESTAMP|timestamp||
+
+You can use other types by specifying `value_type` in `column_options`.
+
+### Example
+
+```yaml
+out:
+  type: db2
+  driver_path: /opt/db2/db2jcc4.jar
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: my_table
+  mode: insert
+```
+
+Advanced configuration:
+
+```yaml
+out:
+  type: db2
+  driver_path: /opt/db2/db2jcc4.jar
+  host: localhost
+  user: myuser
+  password: ""
+  database: my_database
+  table: my_table
+  mode: insert_direct
+  insert_method: native
+  column_options:
+    my_col_1: {type: 'TEXT'}
+    my_col_3: {type: 'INT NOT NULL'}
+    my_col_4: {value_type: string, timestamp_format: `%Y-%m-%d %H:%M:%S %z`, timezone: '-0700'}
+    my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
+```
+
+### Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -0,0 +1,9 @@
+[compileTestJava]*.options*.encoding = 'UTF-8'
+
+dependencies {
+    compile project(':embulk-output-jdbc')
+
+    testCompile 'org.embulk:embulk-standards:0.8.8'
+    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+    testCompile files('driver/db2jcc4.jar')
+}

data/lib/embulk/output/db2.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_output(
+  :db2, "org.embulk.output.DB2OutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/out/test/db2/data/test-char.csv

@@ -0,0 +1,3 @@
+A001,aa,AA,aaaaaaaaaaaa,ああ,いいいい,ａａ,ＡＡ,ａａａａａａａａ
+A002,XXXX,XXXXXXXX,XXXXXXXXXXXXXXXX,XXXX,XXXXXXXX,XXXX,XXXXXXXX,XXXXXXXXXXXXXXXX
+A003,,,,,,,,

data/out/test/db2/data/test-datetime.csv

@@ -0,0 +1,3 @@
+A001,2016/09/08,12:34:45,2016/09/09 12:34:45.123456,2016/09/10 12:34:45,2016/09/11 12:34:45.123456789012
+A002,2016/12/31,23:59:59,2016/12/31 23:59:59.999999,2016/12/31 23:59:59,2016/12/31 23:59:59.999999999999
+A003,,,,,

data/out/test/db2/data/test-number.csv

@@ -0,0 +1,3 @@
+A001,12345,123456789,123456789012,123456.78,876543.21,1.23456,1.23456789012,3.45678901234
+A002,-9999,-999999999,-999999999999,-999999.99,-999999.99,-9.999999,-9.999999,-9.99999999999999
+A003,,,,,,,,

data/out/test/db2/yml/test-insert-direct-char.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: '/db2/data/test-char.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: CHAR_ITEM, type: string}
+    - {name: VARCHAR_ITEM, type: string}
+    - {name: CLOB_ITEM, type: string}
+    - {name: GRAPHIC_ITEM, type: string}
+    - {name: VARGRAPHIC_ITEM, type: string}
+    - {name: NCHAR_ITEM, type: string}
+    - {name: NVARCHAR_ITEM, type: string}
+    - {name: NCLOB_ITEM, type: string}
+out:
+    type: db2
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST_CHAR
+    mode: insert_direct

data/out/test/db2/yml/test-insert-direct-datetime.yml

@@ -0,0 +1,24 @@
+in:
+  type: file
+  path_prefix: '/db2/data/test-datetime.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: DATE_ITEM, type: timestamp, format: '%Y/%m/%d'}
+    - {name: TIME_ITEM, type: timestamp, format: '%H:%M:%S'}
+    - {name: TIMESTAMP_ITEM, type: timestamp, format: '%Y/%m/%d %H:%M:%S.%N'}
+    - {name: TIMESTAMP0_ITEM, type: timestamp, format: '%Y/%m/%d %H:%M:%S'}
+    - {name: TIMESTAMP12_ITEM, type: timestamp, format: '%Y/%m/%d %H:%M:%S.%N'}
+out:
+    type: db2
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST_DATETIME
+    mode: insert_direct

data/out/test/db2/yml/test-insert-direct-number.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: '/db2/data/test-number.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: SMALLINT_ITEM, type: long}
+    - {name: INTEGER_ITEM, type: long}
+    - {name: BIGINT_ITEM, type: long}
+    - {name: DECIMAL_ITEM, type: string}
+    - {name: NUMERIC_ITEM, type: string}
+    - {name: REAL_ITEM, type: double}
+    - {name: DOUBLE_ITEM, type: double}
+    - {name: FLOAT_ITEM, type: double}
+out:
+    type: db2
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST_NUMBER
+    mode: insert_direct

data/src/main/java/org/embulk/output/DB2OutputPlugin.java

@@ -0,0 +1,89 @@
+package org.embulk.output;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.output.db2.DB2BatchInsert;
+import org.embulk.output.db2.DB2OutputConnector;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.MergeConfig;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import static java.util.Locale.ENGLISH;
+
+public class DB2OutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    public interface DB2PluginTask
+            extends PluginTask
+    {
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
+        @Config("host")
+        public String getHost();
+
+        @Config("port")
+        @ConfigDefault("50000")
+        public int getPort();
+
+        @Config("database")
+        public String getDatabase();
+
+        @Config("user")
+        public String getUser();
+
+        @Config("password")
+        @ConfigDefault("null")
+        public Optional<String> getPassword();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return DB2PluginTask.class;
+    }
+
+    @Override
+    protected Features getFeatures(PluginTask task)
+    {
+        return new Features()
+            .setMaxTableNameLength(128) // http://www.ibm.com/support/knowledgecenter/SSEPGG_11.1.0/com.ibm.db2.luw.sql.ref.doc/doc/r0001029.html
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setIgnoreMergeKeys(false);
+    }
+
+    @Override
+    protected DB2OutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        DB2PluginTask db2Task = (DB2PluginTask) task;
+
+        if (db2Task.getDriverPath().isPresent()) {
+            loadDriverJar(db2Task.getDriverPath().get());
+        }
+
+        String url = String.format(ENGLISH, "jdbc:db2://%s:%d/%s",
+                db2Task.getHost(), db2Task.getPort(), db2Task.getDatabase());
+
+        Properties props = new Properties();
+        props.putAll(db2Task.getOptions());
+        props.setProperty("user", db2Task.getUser());
+        if (db2Task.getPassword().isPresent()) {
+            props.setProperty("password", db2Task.getPassword().get());
+        }
+
+        return new DB2OutputConnector(url, props, null);
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
+    {
+        return new DB2BatchInsert(getConnector(task, true), mergeConfig);
+    }
+}

data/src/main/java/org/embulk/output/db2/DB2BatchInsert.java

@@ -0,0 +1,33 @@
+package org.embulk.output.db2;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.StandardBatchInsert;
+
+import com.google.common.base.Optional;
+
+public class DB2BatchInsert
+        extends StandardBatchInsert
+{
+    public DB2BatchInsert(DB2OutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
+    {
+        super(connector, mergeConfig);
+    }
+
+    @Override
+    public void flush() throws IOException, SQLException
+    {
+        try {
+            super.flush();
+
+        } catch (SQLException e) {
+            if (e.getNextException() != null) {
+                // SQLException of DB2 doesn't contain details.
+                throw new SQLException(e.toString(), e.getNextException());
+            }
+            throw e;
+        }
+    }
+}

data/src/main/java/org/embulk/output/db2/DB2OutputConnection.java

@@ -0,0 +1,140 @@
+package org.embulk.output.db2;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcSchema;
+
+import static java.util.Locale.ENGLISH;
+
+public class DB2OutputConnection
+        extends JdbcOutputConnection
+{
+    public DB2OutputConnection(Connection connection, String schemaName, boolean autoCommit)
+            throws SQLException
+    {
+        super(connection, schemaName);
+        connection.setAutoCommit(autoCommit);
+    }
+
+    @Override
+    protected String buildRenameTableSql(String fromTable, String toTable)
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append("RENAME TABLE ");
+        sb.append(quoteIdentifierString(fromTable));
+        sb.append(" TO ");
+        sb.append(quoteIdentifierString(toTable));
+        return sb.toString();
+    }
+
+    @Override
+    protected void setSearchPath(String schema) throws SQLException
+    {
+        // NOP
+    }
+
+    @Override
+    public void dropTableIfExists(String tableName) throws SQLException
+    {
+        if (tableExists(tableName)) {
+            dropTable(tableName);
+        }
+    }
+
+    @Override
+    protected void dropTableIfExists(Statement stmt, String tableName) throws SQLException
+    {
+        if (tableExists(tableName)) {
+            dropTable(stmt, tableName);
+        }
+    }
+
+    @Override
+    public void createTableIfNotExists(String tableName, JdbcSchema schema) throws SQLException
+    {
+        if (!tableExists(tableName)) {
+            createTable(tableName, schema);
+        }
+    }
+
+    public void createTable(String tableName, JdbcSchema schema) throws SQLException
+    {
+        Statement stmt = connection.createStatement();
+        try {
+            String sql = buildCreateTableSql(tableName, schema);
+            executeUpdate(stmt, sql);
+            commitIfNecessary(connection);
+        } catch (SQLException ex) {
+            throw safeRollback(connection, ex);
+        } finally {
+            stmt.close();
+        }
+    }
+
+    protected String buildCreateTableSql(String name, JdbcSchema schema)
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("CREATE TABLE ");
+        quoteIdentifierString(sb, name);
+        sb.append(buildCreateTableSchemaSql(schema));
+        return sb.toString();
+    }
+
+    @Override
+    protected String buildColumnTypeName(JdbcColumn c)
+    {
+        switch(c.getSimpleTypeName()) {
+        case "BOOLEAN":
+            return "SMALLINT";
+
+        // NCHAR/NVARCHAR/NCLOB are synonyms for CHAR/VARCHAR/CLOB/GRAPHIC/VARGRAPHIC
+        case "CHAR":
+        case "VARCHAR":
+        case "CLOB":
+            String charUnit;
+            if (c.getSizeTypeParameter() == c.getDataLength()) {
+                charUnit = "OCTETS";
+            } else if (c.getSizeTypeParameter() * 2 == c.getDataLength()) {
+                charUnit = "CODEUNITS16";
+            } else if (c.getSizeTypeParameter() * 4 == c.getDataLength()) {
+                charUnit = "CODEUNITS32";
+            } else {
+                throw new IllegalArgumentException(String.format(ENGLISH, "Column %s has unexpected size %d and length %d.",
+                        c.getName(), c.getSizeTypeParameter(), c.getDataLength()));
+            }
+            return String.format(ENGLISH, "%s(%d %s)", c.getSimpleTypeName(), c.getSizeTypeParameter(), charUnit);
+
+        case "GRAPHIC":
+        case "VARGRAPHIC":
+            String graphicUnit;
+            if (c.getSizeTypeParameter() == c.getDataLength()) {
+                graphicUnit = "CODEUNITS16";
+            } else if (c.getSizeTypeParameter() * 2 == c.getDataLength()) {
+                graphicUnit = "CODEUNITS32";
+            } else {
+                throw new IllegalArgumentException(String.format(ENGLISH, "Column %s has unexpected size %d and length %d.",
+                        c.getName(), c.getSizeTypeParameter(), c.getDataLength()));
+            }
+            return String.format(ENGLISH, "%s(%d %s)", c.getSimpleTypeName(), c.getSizeTypeParameter(), graphicUnit);
+
+        default:
+            return super.buildColumnTypeName(c);
+        }
+    }
+
+    @Override
+    public void close() throws SQLException
+    {
+        if (!connection.isClosed()) {
+            // DB2 JDBC Driver requires explicit commit/rollback before closing connection.
+            connection.rollback();
+        }
+
+        super.close();
+    }
+}

data/src/main/java/org/embulk/output/db2/DB2OutputConnector.java

@@ -0,0 +1,49 @@
+package org.embulk.output.db2;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.embulk.output.jdbc.JdbcOutputConnector;
+
+public class DB2OutputConnector
+        implements JdbcOutputConnector
+{
+    private final String url;
+    private final Properties properties;
+    private final String schemaName;
+
+    public DB2OutputConnector(String url, Properties properties, String schemaName)
+    {
+        try {
+            Class.forName("com.ibm.db2.jcc.DB2Driver");
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+        this.url = url;
+        this.properties = properties;
+        this.schemaName = schemaName;
+    }
+
+    @Override
+    public DB2OutputConnection connect(boolean autoCommit) throws SQLException
+    {
+        Connection c = DriverManager.getConnection(url, properties);
+        if (c == null) {
+            // driver.connect returns null when url is "jdbc:mysql://...".
+            throw new SQLException("Invalid url : " + url);
+        }
+
+        try {
+            DB2OutputConnection con = new DB2OutputConnection(c, schemaName, autoCommit);
+            c = null;
+            return con;
+
+        } finally {
+            if (c != null) {
+                c.close();
+            }
+        }
+    }
+}