embulk-output-sqlserver 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d95b2b450eb4a21786e7f1395c2920ada8826573
-  data.tar.gz: baa43925d9fa961ba0d0d13cb99ed4d5ec5981e4
+  metadata.gz: 6a2b4419d9125103b0c560157271d356377bf3ed
+  data.tar.gz: e9013fa22cecbc1c82980b2593fb104dc23630da
 SHA512:
-  metadata.gz: 97c35831f60cd782bf275735f9fd73ce875daf791d4a008c7d26ed07e5e44db79454159caf5743c4eec453db3c2adf07a98ee5dd32a31c1185de9aa86dbef3bb
-  data.tar.gz: 4d954ca104379794dd62ae5a55b50092586e88dabb42705ae87879ce54f1795663b2e94870d8b536f1a52e11c0f5b0b7c5515db0d419889e1ab1aae346f2f34b
+  metadata.gz: 1962baaf5f270e572fa52d3e482751270f8f5e1506864146628785f75c459caf790682f6641f32803ce58e012e9f71543a3c2a18c163203e3c999c89a46b2155
+  data.tar.gz: f60384f6964fd889151f82e2c8b49b00b562bd28b10976593fe585f33f0c5c09c2979dcdaab1ae8f917bae6183a8e11e944e7bc013caf28c777939e791567a5f

data/README.md

@@ -10,8 +10,8 @@ SQL Server output plugin for Embulk loads records to SQL Server.
 
 ## Configuration
 
-- **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, open-source driver (jTDS driver) is used (string)
-- **host**: database host name (string, required)
+- **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, the bundled JDBC driver (Microsoft SQL Server JDBC driver 7.2.2) will be used. (string)
+- **driver_type**: the current version of embulk-input-sqlserver will use Microsoft SQL Server JDBC driver in default, but version 0.10.0 or older will use jTDS driver in default. You can still use jTDS driver by setting this option to "jtds". (string, default: "mssql-jdbc")- **host**: database host name (string, required)
 - **port**: database port number (integer, default: 1433)
 - **integratedSecutiry**: whether to use integrated authentication or not. The `sqljdbc_auth.dll` must be located on Java library path if using integrated authentication. : (boolean, default: false)
 ```
@@ -44,11 +44,13 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, insert_truncate and merge modes), when it creates the target table (insert_direct, merge_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
   - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
+  NOTE: the default value_type for DATE, TIME and DATETIME2 is `string`, because jTDS driver, default JDBC driver for older embulk-output-sqlserver, returns Types.VARCHAR as JDBC type for these types.
   - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
   - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
 - **before_load**: if set, this SQL will be executed before loading all records. In truncate_insert mode, the SQL will be executed after truncating. replace mode doesn't support this option.
 - **after_load**: if set, this SQL will be executed after loading all records.
-
+- **connect_timeout**: timeout for the driver to connect. 0 means the default of SQL Server (15 by default). (integer (seconds), optional)
+- **socket_timeout**: timeout for executing the query. 0 means no timeout. (integer (seconds), optional)
 ### Modes
 
 * **insert**:

data/build.gradle

@@ -1,8 +1,8 @@
 dependencies {
     compile project(':embulk-output-jdbc')
+    compile 'com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8'
     compile 'net.sourceforge.jtds:jtds:1.3.1'
 
     testCompile 'org.embulk:embulk-standards:0.8.22'
     testCompile project(':embulk-output-jdbc').sourceSets.test.output
-    testCompile files('test_jdbc_driver/sqljdbc41.jar')
 }

data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java

@@ -29,9 +29,6 @@ import static java.util.Locale.ENGLISH;
 public class SQLServerOutputPlugin
         extends AbstractJdbcOutputPlugin
 {
-    // for test
-    public static boolean preferMicrosoftDriver = true;
-
     private static int DEFAULT_PORT = 1433;
 
     public interface SQLServerPluginTask
@@ -41,6 +38,10 @@ public class SQLServerOutputPlugin
         @ConfigDefault("null")
         public Optional<String> getDriverPath();
 
+        @Config("driver_type")
+        @ConfigDefault("\"mssql-jdbc\"")
+        public String getDriverType();
+
         @Config("host")
         @ConfigDefault("null")
         public Optional<String> getHost();
@@ -92,6 +93,14 @@ public class SQLServerOutputPlugin
         @Config("database_encoding")
         @ConfigDefault("\"MS932\"")
         public String getDatabaseEncoding();
+
+        @Config("connect_timeout")
+        @ConfigDefault("null")
+        public Optional<Integer> getConnectTimeout();
+
+        @Config("socket_timeout")
+        @ConfigDefault("null")
+        public Optional<Integer> getSocketTimeout();
     }
 
     private static class UrlAndProperties {
@@ -134,35 +143,30 @@ public class SQLServerOutputPlugin
     protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
     {
         SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
-        boolean useJtdsDriver = false;
 
         if (sqlServerTask.getDriverPath().isPresent()) {
             addDriverJarToClasspath(sqlServerTask.getDriverPath().get());
+        }
+
+        boolean useJtdsDriver;
+        if (sqlServerTask.getDriverType().equalsIgnoreCase("mssql-jdbc")) {
+            useJtdsDriver = false;
             try {
-                Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-            } catch (Exception e) {
-                throw new ConfigException("Driver set at field 'driver_path' doesn't include Microsoft SQLServerDriver", e);
+                Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
             }
-        } else {
-            boolean useMicrosoftDriver = false;
-            if (preferMicrosoftDriver) {
-                // prefer Microsoft SQLServerDriver if it is in classpath
-                try {
-                    Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-                    useMicrosoftDriver = true;
-                } catch (Exception e) {
-                }
+            catch (Exception e) {
+                throw new ConfigException("Can't load Microsoft SQLServerDriver from classpath", e);
             }
-
-            if (!useMicrosoftDriver) {
-                logger.info("Using jTDS Driver");
-                try {
-                    Class.forName("net.sourceforge.jtds.jdbc.Driver");
-                } catch (Exception e) {
-                    throw new ConfigException("'driver_path' doesn't set and can't find jTDS driver", e);
-                }
-                useJtdsDriver = true;
+        } else if (sqlServerTask.getDriverType().equalsIgnoreCase("jtds")) {
+            useJtdsDriver = true;
+            try {
+                Class.forName("net.sourceforge.jtds.jdbc.Driver").newInstance();
+            }
+            catch (Exception e) {
+                throw new ConfigException("Can't load jTDS Driver from classpath", e);
             }
+        } else {
+            throw new ConfigException("Unknown driver_type : " + sqlServerTask.getDriverType());
         }
 
         UrlAndProperties urlProps = getUrlAndProperties(sqlServerTask, useJtdsDriver);
@@ -191,9 +195,8 @@ public class SQLServerOutputPlugin
 
             if (sqlServerTask.getHost().isPresent()
                     || sqlServerTask.getInstance().isPresent()
-                    || sqlServerTask.getDatabase().isPresent()
-                    || sqlServerTask.getIntegratedSecurity().isPresent()) {
-                throw new IllegalArgumentException("'host', 'port', 'instance', 'database' and 'integratedSecurity' parameters are invalid if 'url' parameter is set.");
+                    || sqlServerTask.getDatabase().isPresent()) {
+                throw new IllegalArgumentException("'host', 'instance' and 'database' parameters are invalid if 'url' parameter is set.");
             }
             url = sqlServerTask.getUrl().get();
         } else {
@@ -232,7 +235,11 @@ public class SQLServerOutputPlugin
             if (!sqlServerTask.getUser().isPresent()) {
                 throw new ConfigException("'user' option is required but not set.");
             }
-        }else {
+
+            if (sqlServerTask.getSocketTimeout().isPresent()) {
+                props.setProperty("socketTimeout", String.valueOf(sqlServerTask.getSocketTimeout().get())); // seconds
+            }
+        } else {
             StringBuilder urlBuilder = new StringBuilder();
             if (sqlServerTask.getInstance().isPresent()) {
                 urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s",
@@ -254,9 +261,18 @@ public class SQLServerOutputPlugin
                     throw new IllegalArgumentException("Field 'password' is not set.");
                 }
             }
+
+            if (sqlServerTask.getSocketTimeout().isPresent()) {
+                props.setProperty("socketTimeout", String.valueOf(sqlServerTask.getSocketTimeout().get() * 1000L)); // milliseconds
+            }
+
             url = urlBuilder.toString();
         }
 
+        if (sqlServerTask.getConnectTimeout().isPresent()) {
+            props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout().get())); // seconds
+        }
+
         return new UrlAndProperties(url, props);
     }
 

data/src/main/java/org/embulk/output/sqlserver/SQLServerOutputConnection.java

@@ -47,6 +47,15 @@ public class SQLServerOutputConnection
             return "TEXT";
         case "TIMESTAMP":
             return "DATETIME2";
+        case "NVARCHAR":
+            if(c.getSizeTypeParameter() > 4000) {
+                return "NVARCHAR(max)";
+            }
+        case "VARCHAR":
+            if(c.getSizeTypeParameter() > 8000) {
+                return "VARCHAR(max)";
+            }
+
         default:
             return super.buildColumnTypeName(c);
         }

data/src/main/java/org/embulk/output/sqlserver/setter/SQLServerColumnSetterFactory.java

@@ -7,6 +7,7 @@ import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcColumnOption;
 import org.embulk.output.jdbc.setter.ColumnSetter;
 import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.jdbc.setter.StringColumnSetter;
 import org.joda.time.DateTimeZone;
 
 public class SQLServerColumnSetterFactory
@@ -42,6 +43,19 @@ public class SQLServerColumnSetterFactory
         case "time":
             return new SQLServerSqlTimeColumnSetter(batch, column, newDefaultValueSetter(column, option), newCalendar(option));
 
+        case "coerce":
+            switch (column.getSimpleTypeName().toLowerCase()) {
+            case "date":
+            case "datetime2":
+            case "time":
+            case "sql_variant":
+            case "datetimeoffset":
+                // because jTDS driver, default JDBC driver for older embulk-output-sqlserver, returns Types.VARCHAR as JDBC type for these types.
+                return new StringColumnSetter(batch, column, newDefaultValueSetter(column, option), newTimestampFormatter(option));
+            default:
+                return super.newColumnSetter(column, option);
+            }
+
         default:
             return super.newColumnSetter(column, option);
         }

data/src/test/java/org/embulk/output/sqlserver/BasicTest.java

@@ -1,6 +1,7 @@
 package org.embulk.output.sqlserver;
 
 import static org.embulk.output.sqlserver.SQLServerTests.execute;
+import static org.embulk.output.sqlserver.SQLServerTests.executeQuery;
 import static org.embulk.output.sqlserver.SQLServerTests.selectRecords;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
@@ -9,7 +10,9 @@ import java.io.File;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.FileSystems;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
@@ -177,18 +180,31 @@ public class BasicTest
         //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
     }
 
+    @Test
+    public void testMax() throws Exception
+    {
+        Path in1 = embulk.createTempFile(".csv");
+        String line3 = "2," + createString('A', 8000) + "," + createString('a', 10000) + "," + createString('あ', 4000) + "," + createString('ア', 10000);
+        Files.write(in1, Arrays.asList("ID:long,C1:string,C2:string,C3:string,C4:string", "1,,,,", line3));
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_max.yml")), in1);
+        assertThat(executeQuery(embulk, "SELECT ID, LEN(C1), LEN(C2), LEN(C3), LEN(C4) FROM TEST_MAX"), is("1,NULL,NULL,NULL,NULL\n2,8000,10000,4000,10000\n"));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private static String createString(char c, int n)
+    {
+        char[] chars = new char[n];
+        Arrays.fill(chars, c);
+        return new String(chars);
+    }
+
     @Test
     public void testJtds() throws Exception
     {
-        SQLServerOutputPlugin.preferMicrosoftDriver = false;
-        try {
-            Path in1 = toPath("test1.csv");
-            TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert.yml")), in1);
-            assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
-            //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
-        } finally {
-            SQLServerOutputPlugin.preferMicrosoftDriver = true;
-        }
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_jtds.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
     }
 
     private Path toPath(String fileName) throws URISyntaxException

data/src/test/java/org/embulk/output/sqlserver/SQLServerTests.java

@@ -37,7 +37,7 @@ public class SQLServerTests
         Integer port = config.get(Integer.class, "port");
         String database = config.get(String.class, "database");
 
-        String url = String.format("jdbc:jtds:sqlserver://%s:%d/%s", host, port, database);
+        String url = String.format("jdbc:sqlserver://%s:%d;databaseName=%s", host, port, database);
 
         return DriverManager.getConnection(url, user, password);
     }
@@ -79,12 +79,17 @@ public class SQLServerTests
     }
 
     public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        return executeQuery(embulk, "SELECT * FROM " + tableName);
+    }
+
+    public static String executeQuery(TestingEmbulk embulk, String query) throws IOException
     {
         Path temp = embulk.createTempFile("txt");
         Files.delete(temp);
 
         // should not use UTF8 because of BOM
-        execute("SET NOCOUNT ON; SELECT * FROM " + tableName, "-h", "-1", "-s", ",", "-W", "-f", "932", "-o", temp.toString());
+        execute("SET NOCOUNT ON; " + query, "-h", "-1", "-s", ",", "-W", "-f", "932", "-o", temp.toString());
 
         List<String> lines = Files.readAllLines(temp, Charset.forName("MS932"));
         Collections.sort(lines);

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/setup.sql

@@ -80,3 +80,13 @@ CREATE TABLE TEST_MERGE2 (
 INSERT INTO TEST_MERGE2 VALUES(10, 20, 'A');
 INSERT INTO TEST_MERGE2 VALUES(10, 21, 'B');
 INSERT INTO TEST_MERGE2 VALUES(11, 20, 'C');
+
+DROP TABLE TEST_MAX
+CREATE TABLE TEST_MAX (
+    ID INT,
+    C1 VARCHAR(8000),
+    C2 VARCHAR(MAX),
+    C3 NVARCHAR(4000),
+    C4 NVARCHAR(MAX),
+    PRIMARY KEY (ID)
+);

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_jtds.yml

@@ -0,0 +1,3 @@
+table: TEST1
+mode: insert
+driver_type: jtds

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_max.yml

@@ -0,0 +1,2 @@
+table: TEST_MAX
+mode: insert

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-10 00:00:00.000000000 Z
+date: 2019-08-22 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,9 +19,10 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.5.jar
-- classpath/embulk-output-sqlserver-0.8.5.jar
+- classpath/embulk-output-jdbc-0.8.6.jar
+- classpath/embulk-output-sqlserver-0.8.6.jar
 - classpath/jtds-1.3.1.jar
+- classpath/mssql-jdbc-7.2.2.jre8.jar
 - lib/embulk/output/sqlserver.rb
 - src/main/java/org/embulk/output/SQLServerOutputPlugin.java
 - src/main/java/org/embulk/output/sqlserver/InsertMethod.java
@@ -45,6 +46,8 @@ files:
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_create_expected.csv
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_direct.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_insert_expected.csv
+- src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_jtds.yml
+- src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_max.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge.csv
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/basic/test_merge_expected.csv