embulk-output-oracle 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 216d1163436a4fe62fe244ab857d45e454a86213
-  data.tar.gz: 95dc5c8ef7de58c37f00973a03ef8d4ca8d059c7
+  metadata.gz: 885c9eec0bf737ff77ed157431bbe16f2db54fbf
+  data.tar.gz: d2044b92a9fcd9148aaf40e3fdb2d2609db6e88e
 SHA512:
-  metadata.gz: bacf819a6d2f2c60dd957310bd03558f4212f24f829301576dd7b50b3ca780e2ad32612b5b76eb4459c8950dbe44bc22875cf6a033c6a6ba74faee280f3d261d
-  data.tar.gz: e1dc2c39559cde175cdbc217b0076b71a759bc5bdaef09429146f783a2497a95d81f2df9de67c6ced280695ed0906a38d5e759bc00a9ea2a1560229f0e1b2039
+  metadata.gz: fa3ca6314c186a4d2d682cc9d6d4db42c68fe4bbb9baae74404c43339d0d222a44a9d62e3a6de56bb1c7a2e18114b048a4cd1f9ba796d88f240a8753f3642c2a
+  data.tar.gz: 07b1d530ca63c91edc7904a1dd6089b131a684ff9525c44acb127150a3785b212f22d8a19185140e815d2b87bd48961ad215fcac97e6d0cfe2cc53683f9438da

data/README.md

@@ -22,12 +22,12 @@ Oracle output plugins for Embulk loads records to Oracle.
 - **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See bellow. (string, required)
 - **insert_method**: see below
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
-- **default_timezone**: If input column type (embulk type) is timestamp and destination column type is `string` or `nstring`, this plugin needs to format the timestamp into a string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
   - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
   - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
-  - **timezone**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. And if the input column type is timestamp and value_type is `date`, this plugin needs to consider timezone. In those cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
 
 ### Modes
 
@@ -153,3 +153,8 @@ For Linux (x64) (only Ubuntu Server 14.04 is tested)
 * OCI_PATH (the directory of Oracle Instant Client Basic and the parent of the "sdk" directory)
 
 (6) Execute src/main/cpp/linux/build.sh .
+
+***
+<img src="https://www.yourkit.com/images/yklogo.png" alt="YourKit"/> is used to improve performance of embulk-output-oracle.  
+YourKit supports open source projects with its full-featured Java Profiler.  
+YourKit, LLC is the creator of <a href="https://www.yourkit.com/java/profiler/index.jsp">YourKit Java Profiler</a> and <a href="https://www.yourkit.com/.net/profiler/index.jsp">YourKit .NET Profiler</a>, innovative and intelligent tools for profiling Java and .NET applications.  

data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java

@@ -2,17 +2,14 @@ package org.embulk.output.oracle;
 
 import java.io.IOException;
 import java.math.BigDecimal;
-import java.sql.Date;
 import java.sql.SQLException;
-import java.sql.Time;
-import java.sql.Timestamp;
 import java.sql.Types;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-
+import java.util.Calendar;
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
@@ -21,6 +18,7 @@ import org.embulk.output.oracle.oci.OCIManager;
 import org.embulk.output.oracle.oci.OCIWrapper;
 import org.embulk.output.oracle.oci.RowBuffer;
 import org.embulk.output.oracle.oci.TableDefinition;
+import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
@@ -67,10 +65,10 @@ public class DirectBatchInsert implements BatchInsert
          * boolean      -> unused
          * byte         -> unused
          * short        -> unused
-         * int          -> unused
-         * long         -> unused
-         * BigDecimal   -> unused
-         * String       -> CHAR,VARCHAR,LONGVARCHAR,CLOB, NCHAR,NVARCHAR,NCLOB, NUMBER
+         * int          -> NUMBER
+         * long         -> NUMBER
+         * BigDecimal   -> NUMBER
+         * String       -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
          * NString      -> unused
          * bytes        -> unused
          * SqlDate      -> unused
@@ -81,7 +79,7 @@ public class DirectBatchInsert implements BatchInsert
 
         formats = new DateFormat[insertSchema.getCount()];
         List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
-        Timestamp dummy = new Timestamp(System.currentTimeMillis());
+        java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
         for (int i = 0; i < insertSchema.getCount(); i++) {
             JdbcColumn insertColumn = insertSchema.getColumn(i);
             switch (insertColumn.getSqlType()) {
@@ -247,7 +245,7 @@ public class DirectBatchInsert implements BatchInsert
     @Override
     public void setBigDecimal(BigDecimal v) throws IOException, SQLException
     {
-        throw new SQLException("Unsupported");
+        buffer.addValue(v);
     }
 
     @Override
@@ -269,21 +267,24 @@ public class DirectBatchInsert implements BatchInsert
     }
 
     @Override
-    public void setSqlDate(Date v, int sqlType) throws IOException, SQLException
+    public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
     {
         throw new SQLException("Unsupported");
     }
 
     @Override
-    public void setSqlTime(Time v, int sqlType) throws IOException, SQLException
+    public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
     {
         throw new SQLException("Unsupported");
     }
 
     @Override
-    public void setSqlTimestamp(Timestamp v, int sqlType) throws IOException, SQLException
+    public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
     {
-        buffer.addValue(formats[buffer.getCurrentColumn()].format(v));
+        java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
+        t.setNanos(v.getNano());
+        DateFormat format = formats[buffer.getCurrentColumn()];
+        format.setTimeZone(calendar.getTimeZone());
+        buffer.addValue(format.format(t));
     }
-
 }

data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java

@@ -18,8 +18,8 @@ public class OracleOutputConnection
 {
     private static final Map<String, String> CHARSET_NAMES = new HashMap<String, String>();
     static {
-        CHARSET_NAMES.put("JA16SJIS", "Shift_JIS");
-        CHARSET_NAMES.put("JA16SJISTILDE", "Shift_JIS");
+        CHARSET_NAMES.put("JA16SJIS", "MS932");
+        CHARSET_NAMES.put("JA16SJISTILDE", "MS932");
         CHARSET_NAMES.put("JA16EUC", "EUC-JP");
         CHARSET_NAMES.put("JA16EUCTILDE", "EUC-JP");
         CHARSET_NAMES.put("AL32UTF8", "UTF-8");

data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java

@@ -1,8 +1,10 @@
 package org.embulk.output.oracle.oci;
 
+import java.math.BigDecimal;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 
+
 public class RowBuffer
 {
     private final TableDefinition table;
@@ -13,7 +15,6 @@ public class RowBuffer
     private int currentPosition = 0;
     private final Charset charset;
 
-
     public RowBuffer(TableDefinition table, int rowCount, Charset charset)
     {
         this.table = table;
@@ -59,6 +60,11 @@ public class RowBuffer
         next();
     }
 
+    public void addValue(BigDecimal value)
+    {
+        addValue(value.toPlainString());
+    }
+
     private void next()
     {
         currentPosition += table.columns[currentColumn].columnSize;

data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java

@@ -89,7 +89,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-insert.yml");
 
-        assertGeneratedTable(table);
+        assertGeneratedTable1(table);
     }
 
     public void testInsertDirect() throws Exception
@@ -151,7 +151,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-replace.yml");
 
-        assertGeneratedTable(table);
+        assertGeneratedTable2(table);
     }
 
     public void testReplaceCreate() throws Exception
@@ -162,7 +162,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-replace.yml");
 
-        assertGeneratedTable(table);
+        assertGeneratedTable2(table);
     }
 
 
@@ -175,7 +175,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-replace-long-name.yml");
 
-        assertGeneratedTable(table);
+        assertGeneratedTable2(table);
     }
 
     public void testReplaceLongNameMultibyte() throws Exception
@@ -184,7 +184,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-replace-long-name-multibyte.yml");
 
-        assertGeneratedTable(table);
+        assertGeneratedTable2(table);
     }
 
     public void testStringTimestamp() throws Exception
@@ -196,7 +196,7 @@ public class OracleOutputPluginTestImpl
 
         run("/yml/test-string-timestamp.yml");
 
-        assertTable(table, TimeZone.getDefault());
+        assertTable(table);
     }
 
     private void dropTable(String table) throws SQLException
@@ -220,12 +220,9 @@ public class OracleOutputPluginTestImpl
 
     private void assertTable(String table) throws Exception
     {
-        assertTable(table, TimeZone.getTimeZone("GMT"));
-    }
-
-
-    private void assertTable(String table, TimeZone timeZone) throws Exception
-    {
+    	// datetime of UTC will be inserted by embulk.
+    	// datetime of default timezone will be selected by JDBC.
+    	TimeZone timeZone = TimeZone.getDefault();
         List<List<Object>> rows = select(table);
 
         /*
@@ -265,13 +262,11 @@ public class OracleOutputPluginTestImpl
         }
     }
 
-    private void assertGeneratedTable(String table) throws Exception
-    {
-        assertGeneratedTable(table, TimeZone.getTimeZone("GMT"));
-    }
-
-    private void assertGeneratedTable(String table, TimeZone timeZone) throws Exception
+    private void assertGeneratedTable1(String table) throws Exception
     {
+    	// datetime of UTC will be inserted by embulk.
+    	// datetime of default timezone will be selected by JDBC.
+    	TimeZone timeZone = TimeZone.getDefault();
         List<List<Object>> rows = select(table);
 
         /*
@@ -311,6 +306,50 @@ public class OracleOutputPluginTestImpl
         }
     }
 
+    private void assertGeneratedTable2(String table) throws Exception
+    {
+    	// datetime of UTC will be inserted by embulk.
+    	// datetime of default timezone will be selected by JDBC.
+    	TimeZone timeZone = TimeZone.getDefault();
+        List<List<Object>> rows = select(table);
+
+        /*
+        A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
+        A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
+        A003,,,,,
+        */
+
+        assertEquals(3, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals("ABCDE", i2.next());
+            assertEquals(new BigDecimal("0"), i2.next());
+            assertEquals(new BigDecimal("123.45"), i2.next());
+            assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
+            assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals("あいうえお", i2.next());
+            assertEquals(new BigDecimal("-9999"), i2.next());
+            assertEquals(new BigDecimal("-99999999.99"), i2.next());
+            assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
+            assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+        }
+    }
+
 
     private Timestamp toTimestamp(String s, TimeZone timeZone)
     {

data/src/test/resources/yml/test-insert-direct.yml

@@ -21,6 +21,6 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
     insert_method: direct
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-insert-oci-split.yml

@@ -21,6 +21,8 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
     insert_method: oci
+    column_options:
+      INTEGER_ITEM: {value_type: pass}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-insert-oci.yml

@@ -21,6 +21,8 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
     insert_method: oci
+    column_options:
+      INTEGER_ITEM: {value_type: pass}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-insert.yml

@@ -21,5 +21,5 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-replace-long-name-multibyte.yml

@@ -22,4 +22,10 @@ out:
     password: test_pw
     table: ＴＥＳＴ12345678901234567890
     mode: replace
+    column_options:
+      VARCHAR2_ITEM: {type: 'VARCHAR2(40)'}
+      INTEGER_ITEM: {type: 'NUMBER(4,0)'}
+      NUMBER_ITEM: {type: 'NUMBER(10,2)'}
+      DATE_ITEM: {type: 'DATE'}
+      TIMESTAMP_ITEM: {type: 'TIMESTAMP'}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-replace-long-name.yml

@@ -22,4 +22,10 @@ out:
     password: test_pw
     table: TEST12345678901234567890123456
     mode: replace
+    column_options:
+      VARCHAR2_ITEM: {type: 'VARCHAR2(40)'}
+      INTEGER_ITEM: {type: 'NUMBER(4,0)'}
+      NUMBER_ITEM: {type: 'NUMBER(10,2)'}
+      DATE_ITEM: {type: 'DATE'}
+      TIMESTAMP_ITEM: {type: 'TIMESTAMP'}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-replace.yml

@@ -22,4 +22,10 @@ out:
     password: test_pw
     table: TEST1
     mode: replace
+    column_options:
+      VARCHAR2_ITEM: {type: 'VARCHAR2(40)'}
+      INTEGER_ITEM: {type: 'NUMBER(4,0)'}
+      NUMBER_ITEM: {type: 'NUMBER(10,2)'}
+      DATE_ITEM: {type: 'DATE'}
+      TIMESTAMP_ITEM: {type: 'TIMESTAMP'}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-string-timestamp.yml

@@ -21,5 +21,8 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
+    column_options:
+      DATE_ITEM: {value_type: pass}
+      TIMESTAMP_ITEM: {value_type: pass}
     #driver_path: driver/ojdbc7.jar

data/src/test/resources/yml/test-url.yml

@@ -19,6 +19,6 @@ out:
     user: TEST_USER
     password: test_pw
     table: TEST1
-    mode: insert
+    mode: insert_direct
     #driver_path: driver/ojdbc7.jar
     url: jdbc:oracle:thin:@localhost:1521:TESTDB

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-oracle
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-06-11 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -67,8 +67,8 @@ files:
 - src/test/resources/yml/test-replace.yml
 - src/test/resources/yml/test-string-timestamp.yml
 - src/test/resources/yml/test-url.yml
-- classpath/embulk-output-jdbc-0.3.0.jar
-- classpath/embulk-output-oracle-0.3.0.jar
+- classpath/embulk-output-jdbc-0.4.0.jar
+- classpath/embulk-output-oracle-0.4.0.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0