embulk-output-sqlserver 0.7.11 → 0.7.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8d8e2c0bbd2a1255d5825697f0c6f2e893d9753
-  data.tar.gz: 14ee14826f9dea6d9d5549e7bdaa41bcb01d8943
+  metadata.gz: e3ed69cc6fcf1d2f3c438c92cef31529785ca59b
+  data.tar.gz: 9918999eda849dadc22bd7e49941b0f95e41fb48
 SHA512:
-  metadata.gz: 913d8c0c5b02f0ef7488c3063b0e022b606774c6c0856d2a78670e2119a5b7708eb88efa1f1e47e4cbe9014ecf448eb0a867bc528d6ef76e67c4f493658a437c
-  data.tar.gz: df036fa9b4a164a18238f56273ba578d962bc1f43d97037e9dc740aa0517fedaf624bef35443631e5df7b8afeb728712aa5d90473e38ac0ed73ec52f546261e0
+  metadata.gz: 2afd308537cba8f93859abaaf3e0b7058ac5fa160827062f2a76fbf491fd367224bf082c14a3ffc71639bf0b24da2fe793113d5181edbdc5d523e380ddef93b8
+  data.tar.gz: 6f56d6d312892061f81a463bb80cbdadd008c8613825d197befbba0c140e1038423a1bc0b27071d109ee23729aa3ac129e760079fe9079fe8f569aab49509144

data/README.md

@@ -34,6 +34,8 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
 - **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = T.foo + S.foo` (`T` means target table and `S` means source table). (string array, default: always overwrites with new values)
 - **insert_method**: see below
+- **native_driver**: driver name when using `insert_method: native`. (string, default: `{SQL Server Native Client 11.0}`)
+- **database_encoding**: database encoding when using `insert_method: native`. (string, default: `MS932`)
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -74,7 +76,9 @@ insert_method supports three options.
 "normal" means normal insert (default). It requires SQL Server JDBC driver.
 
 "native" means bulk insert using native client. It is faster than "normal".
-It requires both SQL Server JDBC driver and SQL Server Native Client (11.0).
+It requires both SQL Server JDBC driver and SQL Server Native Client (or newer version of Microsoft ODBC Driver).
+
+If you embulk use on Linux, you needed to [install mssql-tools](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server).
 
 ### Supported types
 

data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java

@@ -83,6 +83,14 @@ public class SQLServerOutputPlugin
         @Config("insert_method")
         @ConfigDefault("\"normal\"")
         public InsertMethod getInsertMethod();
+
+        @Config("native_driver")
+        @ConfigDefault("null")
+        public Optional<String> getNativeDriverName();
+
+        @Config("database_encoding")
+        @ConfigDefault("\"MS932\"")
+        public String getDatabaseEncoding();
     }
 
     private static class UrlAndProperties {
@@ -265,8 +273,10 @@ public class SQLServerOutputPlugin
     {
         SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
         if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
-            return new NativeBatchInsert(sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
-                    sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword());
+            return new NativeBatchInsert(
+                    sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
+                    sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword(),
+                    sqlServerTask.getNativeDriverName(), sqlServerTask.getDatabaseEncoding());
         }
         return new StandardBatchInsert(getConnector(task, true), mergeConfig);
     }

data/src/main/java/org/embulk/output/sqlserver/NativeBatchInsert.java

@@ -33,6 +33,8 @@ public class NativeBatchInsert implements BatchInsert
     private final String database;
     private final Optional<String> user;
     private final Optional<String> password;
+    private final Optional<String> nativeDriverName;
+    private final String databaseEncoding;
 
     private int batchWeight;
     private int batchRows;
@@ -45,7 +47,8 @@ public class NativeBatchInsert implements BatchInsert
 
 
     public NativeBatchInsert(String server, int port, Optional<String> instance,
-            String database, Optional<String> user, Optional<String> password)
+            String database, Optional<String> user, Optional<String> password,
+            Optional<String> nativeDriverName, String databaseEncoding)
     {
         this.server = server;
         this.port = port;
@@ -53,7 +56,8 @@ public class NativeBatchInsert implements BatchInsert
         this.database = database;
         this.user = user;
         this.password = password;
-
+        this.nativeDriverName = nativeDriverName;
+        this.databaseEncoding = databaseEncoding;
         lastColumnIndex = 0;
     }
 
@@ -62,8 +66,7 @@ public class NativeBatchInsert implements BatchInsert
     public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
     {
         columnCount = insertSchema.getCount();
-        client.open(server, port, instance, database, user, password, loadTable.getTableName());
-
+        client.open(server, port, instance, database, user, password, loadTable.getTableName(), nativeDriverName, databaseEncoding);
         formats = new DateFormat[insertSchema.getCount()];
         for (int i = 0; i < insertSchema.getCount(); i++) {
             JdbcColumn column = insertSchema.getColumn(i);

data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClientWrapper.java

@@ -8,6 +8,7 @@ import java.util.HashMap;
 import java.util.Map;
 
 import jnr.ffi.LibraryLoader;
+import jnr.ffi.Platform;
 import jnr.ffi.Pointer;
 import jnr.ffi.Runtime;
 import jnr.ffi.provider.jffi.ArrayMemoryIO;
@@ -33,35 +34,46 @@ public class NativeClientWrapper
 
     public NativeClientWrapper()
     {
+        Platform platform = Platform.getPlatform();
+        Platform.OS os = platform.getOS();
+        String odbcLibName;
+        String nativeClientLibName;
+        if (os == Platform.OS.WINDOWS) {
+          odbcLibName = "odbc32";
+          nativeClientLibName = "sqlncli11";
+        } else {
+          odbcLibName = "odbc";
+          nativeClientLibName = "msodbcsql";
+        }
         synchronized (NativeClientWrapper.class) {
             if (odbc == null) {
-                logger.info("Loading SQL Server Native Client library (odbc32).");
+                logger.info(String.format("Loading SQL Server Native Client library (%s).", odbcLibName));
                 try {
-                    odbc = LibraryLoader.create(ODBC.class).failImmediately().load("odbc32");
+                    odbc = LibraryLoader.create(ODBC.class).failImmediately().load(odbcLibName);
                 } catch (UnsatisfiedLinkError e) {
-                    throw new RuntimeException("odbc32.dll not found.", e);
+                    throw new RuntimeException(platform.mapLibraryName(odbcLibName) + " not found.", e);
                 }
             }
             if (client == null) {
-                logger.info("Loading SQL Server Native Client library (sqlncli11).");
+                logger.info(String.format("Loading SQL Server Native Client library (%s).", nativeClientLibName));
                 try {
-                    client = LibraryLoader.create(NativeClient.class).failImmediately().load("sqlncli11");
+                    client = LibraryLoader.create(NativeClient.class).failImmediately().load(nativeClientLibName);
                 } catch (UnsatisfiedLinkError e) {
-                    throw new RuntimeException("sqlncli11.dll not found.", e);
+                    throw new RuntimeException(platform.mapLibraryName(nativeClientLibName) + " not found.", e);
                 }
             }
         }
-
-        charset = Charset.forName("MS932");
-        wideCharset = Charset.forName("UTF-16LE");
     }
 
     public void open(String server, int port, Optional<String> instance,
             String database, Optional<String> user, Optional<String> password,
-            String table)
+            String table, Optional<String> nativeDriverName,
+            String databaseEncoding)
                     throws SQLException
     {
         // environment handle
+        charset = Charset.forName(databaseEncoding);
+        wideCharset = Charset.forName("UTF-16LE");
         Pointer envHandlePointer = createPointerPointer();
         checkSQLResult("SQLAllocHandle(SQL_HANDLE_ENV)", odbc.SQLAllocHandle(
                 ODBC.SQL_HANDLE_ENV,
@@ -92,7 +104,11 @@ public class NativeClientWrapper
                 ODBC.SQL_IS_INTEGER));
 
         StringBuilder connectionString = new StringBuilder();
-        connectionString.append("Driver={SQL Server Native Client 11.0};");
+        if (nativeDriverName.isPresent()) {
+            connectionString.append(String.format("Driver=%s;", nativeDriverName.get()));
+        } else {
+            connectionString.append("Driver={SQL Server Native Client 11.0};");
+        }
         if (instance.isPresent()) {
             connectionString.append(String.format("Server=%s,%d\\%s;", server, port, instance.get()));
         } else {
@@ -152,14 +168,15 @@ public class NativeClientWrapper
     public int bindValue(int columnIndex, String value) throws SQLException
     {
         ByteBuffer bytes = charset.encode(value);
-        Pointer pointer = prepareBuffer(columnIndex, bytes.remaining());
-        pointer.put(0, bytes.array(), 0, bytes.remaining());
+        int size = bytes.remaining();
+        Pointer pointer = prepareBuffer(columnIndex, size);
+        pointer.put(0, bytes.array(), 0, size);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLCHARACTER,
@@ -169,14 +186,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, boolean value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 1);
+        int size = 1;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putByte(0, value ? (byte)1 : (byte)0);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLBIT,
@@ -186,14 +204,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, byte value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 1);
+        int size = 1;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putByte(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLINT1,
@@ -203,14 +222,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, short value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 2);
+        int size = 2;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putShort(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLINT2,
@@ -220,14 +240,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, int value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 4);
+        int size = 4;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putInt(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLINT4,
@@ -237,14 +258,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, long value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 8);
+        int size = 8;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putLongLong(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLINT8,
@@ -254,14 +276,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, float value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 4);
+        int size = 4;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putFloat(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLFLT4,
@@ -271,14 +294,15 @@ public class NativeClientWrapper
 
     public int bindValue(int columnIndex, double value) throws SQLException
     {
-        Pointer pointer = prepareBuffer(columnIndex, 8);
+        int size = 8;
+        Pointer pointer = prepareBuffer(columnIndex, size);
         pointer.putDouble(0, value);
 
         checkBCPResult("bcp_bind", client.bcp_bind(
                 odbcHandle,
                 pointer,
                 0,
-                (int)pointer.size(),
+                size,
                 null,
                 0,
                 NativeClient.SQLFLT8,

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test1.csv

@@ -2,3 +2,4 @@ ID:string,TINYINT_ITEM:long,SMALLINT_ITEM:long,INT_ITEM:long,BIGINT_ITEM:long,BI
 A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900
 A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000,-999999999999000000,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-31 00:00:00.0 +0900,2017-12-31 23:59:59.997 +0900,2018-12-31 23:59:59.9999999 +0900,2019-12-31 23:59:59.99 +0900,2020-12-31 23:59:59.0 +0900,1970-01-01 23:59:59.9999999 +0900,1970-01-01 23:59:59.99 +0900
 A003,,,,,,,,,,,,,,,,,,,,,,,,
+A004,9,9,9,9,0,9,9,9,9,9,9,x,x,x,X,X,X,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_create_expected.csv

@@ -1,3 +1,4 @@
 A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000
 A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000.0,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-30 15:00:00.0000000,2017-12-31 14:59:59.9970000,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.9900000,2020-12-31 14:59:59.0000000,1970-01-01 14:59:59.9999990,1970-01-01 14:59:59.9900000
 A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
+A004,9,9,9,9,0,9,9,9,9,9.0,9.0,x,x,x,X,X,X,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_expected.csv

@@ -2,3 +2,4 @@
 A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a   ,b,c,A   ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
 A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき  ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99
 A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
+A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x   ,x,x,X   ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_truncate_insert_expected.csv

@@ -1,3 +1,4 @@
 A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a   ,b,c,A   ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
 A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき  ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99
 A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
+A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x   ,x,x,X   ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.7.11
+  version: 0.7.12
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-11-24 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.11.jar
-- classpath/embulk-output-sqlserver-0.7.11.jar
+- classpath/embulk-output-jdbc-0.7.12.jar
+- classpath/embulk-output-sqlserver-0.7.12.jar
 - classpath/jtds-1.3.1.jar
 - lib/embulk/output/sqlserver.rb
 - src/main/java/org/embulk/output/SQLServerOutputPlugin.java