embulk-output-oracle 0.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cc81277bd2a1f2ae80924bb543d6eecfcd3d4d6c
+  data.tar.gz: 3b0b5fa66bc6429a37b1ce15bd3a1d9ea4aca237
+SHA512:
+  metadata.gz: 215e1945daf0792754efdfb7c6adc1596d9a020a8dd53584367be82f9a9d44397a77ae42cbd2601f78b1290e4d2afab22a05c9c6a3fd6e47e6c900dad3484231
+  data.tar.gz: 4078b17328554d4abe9bbfa89aab60d109fe6acd29c7b1c1011068375a6e879a7afe6b4c18ac34b4b0e5581be2a875dc4485c320f8aa032362543362b8e762f3

data/README.md

@@ -0,0 +1,46 @@
+# Oracle output plugins for Embulk
+
+Oracle output plugins for Embulk loads records to Oracle.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depnds on the mode:
+  * **insert**: no
+  * **replace**: yes
+* **Resume supported**: no
+
+## Configuration
+
+- **driver_path**: path to the jar file of the Oracle JDBC driver (string)
+- **host**: database host name (string, required if url is not set)
+- **port**: database port number (integer, default: 1521)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required if url is not set)
+- **url**: URL of the JDBC connection (string, optional)
+- **table**: destination table name (string, required)
+- **mode**: "replace" or "insert" (string, required)
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **options**: extra connection properties (hash, default: {})
+
+
+### Example
+
+```yaml
+out:
+  type: oracle
+  driver_path: /opt/oracle/ojdbc6.jar
+  host: localhost
+  user: root
+  password: ""
+  database: my_database
+  table: my_table
+  mode: insert
+```
+
+### Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    compile project(':embulk-output-jdbc')
+}

data/lib/embulk/output/oracle.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_output(
+  :oracle, "org.embulk.output.OracleOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/OracleOutputPlugin.java

@@ -0,0 +1,151 @@
+package org.embulk.output;
+
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.Properties;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigDefault;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.StandardBatchInsert;
+import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.oracle.OracleOutputConnection;
+import org.embulk.output.oracle.OracleOutputConnector;
+import org.embulk.output.oracle.setter.OracleColumnSetterFactory;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.TimestampFormatter;
+
+public class OracleOutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    private static final int MAX_TABLE_NAME_LENGTH = 30;
+
+    public interface OraclePluginTask
+            extends PluginTask
+    {
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
+        @Config("host")
+        @ConfigDefault("null")
+        public Optional<String> getHost();
+
+        @Config("port")
+        @ConfigDefault("1521")
+        public int getPort();
+
+        @Config("database")
+        @ConfigDefault("null")
+        public Optional<String> getDatabase();
+
+        @Config("url")
+        @ConfigDefault("null")
+        public Optional<String> getUrl();
+
+        @Config("user")
+        public String getUser();
+
+        @Config("password")
+        @ConfigDefault("\"\"")
+        public String getPassword();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return OraclePluginTask.class;
+    }
+
+    @Override
+    protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        OraclePluginTask oracleTask = (OraclePluginTask) task;
+
+        if (oracleTask.getDriverPath().isPresent()) {
+            loadDriverJar(oracleTask.getDriverPath().get());
+        }
+
+        String url;
+        if (oracleTask.getUrl().isPresent()) {
+            if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
+                throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
+            }
+
+            url = oracleTask.getUrl().get();
+        } else {
+            if (!oracleTask.getHost().isPresent()) {
+                throw new IllegalArgumentException("Field 'host' is not set.");
+            }
+            if (!oracleTask.getDatabase().isPresent()) {
+                throw new IllegalArgumentException("Field 'database' is not set.");
+            }
+
+            url = String.format("jdbc:oracle:thin:@%s:%d:%s",
+                    oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
+        }
+
+        Properties props = new Properties();
+        props.setProperty("user", oracleTask.getUser());
+        props.setProperty("password", oracleTask.getPassword());
+        props.putAll(oracleTask.getOptions());
+
+        return new OracleOutputConnector(url, props);
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
+    {
+        return new StandardBatchInsert(getConnector(task, true));
+    }
+
+    @Override
+    protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
+            TimestampFormatter timestampFormatter)
+    {
+        return new OracleColumnSetterFactory(batch, pageReader, timestampFormatter);
+    }
+
+    @Override
+    protected String generateSwapTableName(PluginTask task) throws SQLException
+    {
+        return generateSwapTableName(task, "_bl_tmp", MAX_TABLE_NAME_LENGTH);
+    }
+
+    // TODO move this method to AbstractJdbcOutputPlugin
+    protected String generateSwapTableName(PluginTask task, String suffix, int maxTableNameLength) throws SQLException
+    {
+        String tableName = task.getTable();
+        String uniqueSuffix = getTransactionUniqueName() + suffix;
+
+        if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {  // + 1 for '_'
+            // truncate transaction unique name
+            int suffixLength = Math.max(maxTableNameLength - tableName.length() - 1, suffix.length() + 8);  // include 8 characters of the transaction name at least
+            uniqueSuffix = uniqueSuffix.substring(uniqueSuffix.length() - suffixLength);
+        }
+
+        if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {
+            // use truncated table name
+            int truncLength = maxTableNameLength - uniqueSuffix.length() - 1;
+            while (true) {
+                truncLength--;
+                if (truncLength <= 0) {
+                    throw new ConfigException("Table name is too long to generate temporary table name");
+                }
+                tableName = tableName.substring(0, truncLength);
+                //if (!connection.tableExists(tableName)) {
+                    // TODO this doesn't help. Rather than truncating more characters,
+                    //      here needs to replace characters with random characters. But
+                    //      to make the result deterministic. So, an idea is replacing
+                    //      the last character to the first (second, third, ... for each loop)
+                    //      of md5(original table name).
+                    return tableName + "_" + uniqueSuffix;
+                //}
+            }
+        }
+
+        return tableName + "_" + uniqueSuffix;
+    }
+}

data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java

@@ -0,0 +1,75 @@
+package org.embulk.output.oracle;
+
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcSchema;
+
+public class OracleOutputConnection
+        extends JdbcOutputConnection
+{
+    public OracleOutputConnection(Connection connection, boolean autoCommit)
+            throws SQLException
+    {
+        super(connection, getSchema(connection));
+        connection.setAutoCommit(autoCommit);
+    }
+
+    @Override
+    protected String convertTypeName(String typeName)
+    {
+        switch(typeName) {
+        case "BIGINT":
+            return "NUMBER(19,0)";
+        default:
+            return typeName;
+        }
+    }
+
+    @Override
+    protected void setSearchPath(String schema) throws SQLException {
+        // NOP
+    }
+
+
+    @Override
+    public void dropTableIfExists(String tableName) throws SQLException
+    {
+        if (tableExists(tableName)) {
+            dropTable(tableName);
+        }
+    }
+
+    @Override
+    protected void dropTableIfExists(Statement stmt, String tableName) throws SQLException {
+        if (tableExists(tableName)) {
+            dropTable(stmt, tableName);
+        }
+    }
+
+    @Override
+    public void createTableIfNotExists(String tableName, JdbcSchema schema) throws SQLException
+    {
+        if (!tableExists(tableName)) {
+            createTable(tableName, schema);
+        }
+    }
+
+    private static String getSchema(Connection connection) throws SQLException
+    {
+        // Because old Oracle JDBC drivers don't support Connection#getSchema method.
+        String sql = "SELECT SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') FROM DUAL";
+        try (Statement statement = connection.createStatement()) {
+            try (ResultSet resultSet = statement.executeQuery(sql)) {
+                if (resultSet.next()) {
+                    return resultSet.getString(1);
+                }
+                throw new SQLException(String.format("Cannot get schema becase \"%s\" didn't return any value.", sql));
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java

@@ -0,0 +1,47 @@
+package org.embulk.output.oracle;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import org.embulk.output.jdbc.JdbcOutputConnector;
+
+public class OracleOutputConnector
+        implements JdbcOutputConnector
+{
+    private final String url;
+    private final Properties properties;
+
+    public OracleOutputConnector(String url, Properties properties)
+    {
+        try {
+        	Class.forName("oracle.jdbc.OracleDriver");
+        } catch (Exception ex) {
+            throw new RuntimeException(ex);
+        }
+        this.url = url;
+        this.properties = properties;
+    }
+
+    @Override
+    public OracleOutputConnection connect(boolean autoCommit) throws SQLException
+    {
+        Connection c = DriverManager.getConnection(url, properties);
+        if (c == null) {
+        	// driver.connect returns null when url is "jdbc:mysql://...".
+        	throw new SQLException("Invalid url : " + url);
+        }
+
+        try {
+            OracleOutputConnection con = new OracleOutputConnection(c, autoCommit);
+            c = null;
+            return con;
+
+        } finally {
+            if (c != null) {
+                c.close();
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java

@@ -0,0 +1,31 @@
+package org.embulk.output.oracle.setter;
+
+import java.sql.Types;
+
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.setter.ColumnSetter;
+import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.jdbc.setter.StringColumnSetter;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.time.TimestampFormatter;
+
+public class OracleColumnSetterFactory extends ColumnSetterFactory
+{
+	public OracleColumnSetterFactory(BatchInsert batch, PageReader pageReader,
+			TimestampFormatter timestampFormatter)
+	{
+		super(batch, pageReader, timestampFormatter);
+	}
+	
+	@Override
+	public ColumnSetter newColumnSetter(JdbcColumn column)
+    {
+		switch (column.getSqlType()) {
+			case Types.DECIMAL:
+	            return new StringColumnSetter(batch, pageReader, column, timestampFormatter);
+            default:
+        		return super.newColumnSetter(column);
+		}
+	}
+}

metadata

@@ -0,0 +1,53 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-oracle
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-03-15 00:00:00.000000000 Z
+dependencies: []
+description: Inserts or updates records to a table.
+email:
+- frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- build.gradle
+- lib/embulk/output/oracle.rb
+- src/main/java/org/embulk/output/OracleOutputPlugin.java
+- src/main/java/org/embulk/output/oracle/OracleOutputConnection.java
+- src/main/java/org/embulk/output/oracle/OracleOutputConnector.java
+- src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java
+- classpath/embulk-output-jdbc-0.2.1.jar
+- classpath/embulk-output-oracle-0.2.1.jar
+homepage: https://github.com/embulk/embulk-output-jdbc
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: JDBC output plugin for Embulk
+test_files: []