embulk-input-sqlserver 0.10.0 → 0.10.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 22b1735cc828641cff2cbdaf378cff0ba09258dd9b56029c9c10cf3ed49369b9
-  data.tar.gz: 1a29fa810ebf240722d13e668cf1ad7bec43e146a4086d200dbbd04c2004e6ae
+  metadata.gz: 04e9c70944217bbd26e88379fd303c5671420aa56a6dd514c690f0edd948ee14
+  data.tar.gz: d718ebd86e88a4d45d4982c744738aacb6296cfef8e3da45f42ef60261e42085
 SHA512:
-  metadata.gz: 9a1c12af23925060435e6dee86251b6c7a475844186e91a8274f9d9da4a641bf069a272c3688fed040b9ebd73068e4f11e8b735eac442a5d365c2e736723ba11
-  data.tar.gz: 7750ffb7e28628d6fccf80661cc29c158b734a65211ac4bbe3b0585495f7c7cdc93ea5f23fa7dae602c1b674fc24644c4e0ebbc422e2747f05b1e52324623529
+  metadata.gz: dfc3d01c225d6b3a8658d0d7622883e5d0b417b5f44f681a442441808c247f253cd200795354aab7e3a02c984acbd12351ecb072947b64e4864411c360a80280
+  data.tar.gz: 5fdce20cab127d48be9fbb005ff913dc177d804aaf15632c2a0a14dd33df634962aa6a4163d274ba09e2da21f397761ba5aa619647f53453d35e733579a850e3

data/README.md

@@ -9,7 +9,8 @@ SQL Server input plugin for Embulk loads records from SQL Server.
 
 ## Configuration
 
-- **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, open-source driver (jTDS driver) is used (string)
+- **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, the bundled JDBC driver (Microsoft SQL Server JDBC driver 7.2.2) will be used. (string)
+- **driver_type**: the current version of embulk-input-sqlserver will use Microsoft SQL Server JDBC driver in default, but version 0.10.0 or older will use jTDS driver in default. You can still use jTDS driver by setting this option to "jtds". (string, default: "mssql-jdbc")
 - **host**: database host name (string, required if url is not set)
 - **port**: database port number (integer, default: 1433)
 - **integratedSecutiry**: whether to use integrated authentication or not. The `sqljdbc_auth.dll` must be located on Java library path if using integrated authentication. : (boolean, default: false)
@@ -38,13 +39,14 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **socket_timeout**: timeout for executing the query. 0 means no timeout. (integer (seconds), default: 1800)
 - **options**: extra JDBC properties (hash, default: {})
 - **incremental**: if true, enables incremental loading. See next section for details (boolean, default: false)
-- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys)
+- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys). Columns of integer types, string types and `datetime2` are supported.
 - **last_record**: values of the last record for incremental loading (array of objects, default: load all records)
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **default_column_options**: advanced: column_options for each JDBC type as default. key-value pairs where key is a JDBC type (e.g. 'DATE', 'BIGINT') and value is same as column_options's value.
 - **column_options**: advanced: key-value pairs where key is a column name and value is options for the column.
   - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
   (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`)
+  NOTE: the default value_type for DATE, TIME and DATETIME2 is `string`, because jTDS driver, default JDBC driver for older embulk-input-sqlserver, returns Types.VARCHAR as JDBC type for these types.
   - **type**: Column values are converted to this embulk type.
   Available values options are: `boolean`, `long`, `double`, `string`, `json`, `timestamp`).
   By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
@@ -71,13 +73,13 @@ ORDER BY updated_at, id
 
 When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
 
-At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
+At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12.4876590", 5291]` is set,
 
 ```
 SELECT * FROM (
   ...original query is here...
 )
-WHERE updated_at > '2017-01-01 00:32:12' OR (updated_at = '2017-01-01 00:32:12' AND id > 5291)
+WHERE updated_at > '2017-01-01 00:32:12.4876590' OR (updated_at = '2017-01-01 00:32:12.4876590' AND id > 5291)
 ORDER BY updated_at, id
 ```
 

data/build.gradle

@@ -1,5 +1,6 @@
 dependencies {
     compile project(':embulk-input-jdbc')
+    compile 'com.microsoft.sqlserver:mssql-jdbc:7.2.2.jre8'
     compile 'net.sourceforge.jtds:jtds:1.3.1'
 
     testCompile 'org.embulk:embulk-standards:0.9.12'

data/src/main/java/org/embulk/input/SQLServerInputPlugin.java

@@ -33,6 +33,10 @@ public class SQLServerInputPlugin
         @ConfigDefault("null")
         public Optional<String> getDriverPath();
 
+        @Config("driver_type")
+        @ConfigDefault("\"mssql-jdbc\"")
+        public String getDriverType();
+
         @Config("host")
         @ConfigDefault("null")
         public Optional<String> getHost();
@@ -113,26 +117,29 @@ public class SQLServerInputPlugin
         SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
 
         Driver driver;
-        boolean useJtdsDriver = false;
         if (sqlServerTask.getDriverPath().isPresent()) {
             addDriverJarToClasspath(sqlServerTask.getDriverPath().get());
+        }
+
+        boolean useJtdsDriver;
+        if (sqlServerTask.getDriverType().equalsIgnoreCase("mssql-jdbc")) {
+            useJtdsDriver = false;
             try {
                 driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
             }
             catch (Exception e) {
-                throw new ConfigException("Driver set at field 'driver_path' doesn't include Microsoft SQLServerDriver", e);
+                throw new ConfigException("Can't load Microsoft SQLServerDriver from classpath", e);
             }
-        }
-        else {
-            // prefer Microsoft SQLServerDriver if it is in classpath
+        } else if (sqlServerTask.getDriverType().equalsIgnoreCase("jtds")) {
+            useJtdsDriver = true;
             try {
-                driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
+                driver = (Driver) Class.forName("net.sourceforge.jtds.jdbc.Driver").newInstance();
             }
-            catch (Exception ex) {
-                logger.info("Using jTDS Driver");
-                driver = new net.sourceforge.jtds.jdbc.Driver();
-                useJtdsDriver = true;
+            catch (Exception e) {
+                throw new ConfigException("Can't load jTDS Driver from classpath", e);
             }
+        } else {
+            throw new ConfigException("Unknown driver_type : " + sqlServerTask.getDriverType());
         }
 
         UrlAndProperties urlAndProps = buildUrlAndProperties(sqlServerTask, useJtdsDriver);
@@ -141,18 +148,21 @@ public class SQLServerInputPlugin
         props.putAll(sqlServerTask.getOptions());
         logConnectionProperties(urlAndProps.getUrl(), props);
 
-        Connection con = driver.connect(urlAndProps.getUrl(), props);
-        try {
-            SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull(),
-                                                                      sqlServerTask.getTransactionIsolationLevel().orNull());
-            con = null;
-            return c;
-        }
-        finally {
-            if (con != null) {
-                con.close();
+        if (driver != null) {
+            Connection con = driver.connect(urlAndProps.getUrl(), props);
+            try {
+                SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull(),
+                        sqlServerTask.getTransactionIsolationLevel().orNull());
+                con = null;
+                return c;
+            }
+            finally {
+                if (con != null) {
+                    con.close();
+                }
             }
         }
+        throw new ConfigException("Fail to create new connection");
     }
 
     @Override
@@ -166,7 +176,6 @@ public class SQLServerInputPlugin
         Properties props = new Properties();
 
         // common properties
-
         if (sqlServerTask.getUser().isPresent()) {
             props.setProperty("user", sqlServerTask.getUser().get());
         }
@@ -185,6 +194,7 @@ public class SQLServerInputPlugin
         else {
             // SQLServerDriver properties
             props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
+            props.setProperty("socketTimeout", String.valueOf(sqlServerTask.getSocketTimeout() * 1000L)); // milliseconds
 
             props.setProperty("applicationName", sqlServerTask.getApplicationName());
 
@@ -196,9 +206,8 @@ public class SQLServerInputPlugin
         if (sqlServerTask.getUrl().isPresent()) {
             if (sqlServerTask.getHost().isPresent()
                     || sqlServerTask.getInstance().isPresent()
-                    || sqlServerTask.getDatabase().isPresent()
-                    || sqlServerTask.getIntegratedSecurity()) {
-                throw new ConfigException("'host', 'port', 'instance', 'database' and 'integratedSecurity' options are invalid if 'url' option is set.");
+                    || sqlServerTask.getDatabase().isPresent()) {
+                throw new ConfigException("'host', 'instance' and 'database' options are invalid if 'url' option is set.");
             }
 
             return new UrlAndProperties(sqlServerTask.getUrl().get(), props);
@@ -240,7 +249,7 @@ public class SQLServerInputPlugin
             }
         }
         else {
-            // SQLServerDriver URL: host:port[;databaseName=] or host\instance[;databaseName=]
+            // SQLServerDriver URL: host:port[;databaseName=database] or host\instance[;databaseName=database]
             // host\instance:port[;databaseName] is allowed but \instance will be ignored in this case.
             if (sqlServerTask.getInstance().isPresent()) {
                 if (sqlServerTask.getPort() != DEFAULT_PORT) {
@@ -252,7 +261,7 @@ public class SQLServerInputPlugin
                 url = String.format(ENGLISH, "jdbc:sqlserver://%s:%d", sqlServerTask.getHost().get(), sqlServerTask.getPort());
             }
 
-            // ;databaseName=
+            // ;databaseName=database
             if (sqlServerTask.getDatabase().isPresent()) {
                 props.setProperty("databaseName", sqlServerTask.getDatabase().get());
             }
@@ -263,6 +272,7 @@ public class SQLServerInputPlugin
                     throw new ConfigException("'user' options are invalid if 'integratedSecutiry' option is set.");
                 }
                 props.setProperty("integratedSecurity", "true");
+                props.setProperty("authenticationScheme", "JavaKerberos");
             }
             else {
                 if (!sqlServerTask.getUser().isPresent()) {

data/src/main/java/org/embulk/input/sqlserver/getter/SQLServerColumnGetterFactory.java

@@ -18,12 +18,22 @@ public class SQLServerColumnGetterFactory extends ColumnGetterFactory {
     @Override
     public ColumnGetter newColumnGetter(JdbcInputConnection con, AbstractJdbcInputPlugin.PluginTask task, JdbcColumn column, JdbcColumnOption option)
     {
-        ColumnGetter getter = super.newColumnGetter(con, task, column, option);
         switch (column.getTypeName()) {
-            case "datetime":
-                return new TimestampWithoutTimeZoneIncrementalHandler(getter);
-            default:
-                return getter;
+        case "date":
+        case "datetime2":
+        case "time":
+        case "sql_variant":
+        // DateTimeOffset is available only in MSSQL
+        case "datetimeoffset":
+            // because jTDS driver, default JDBC driver for older embulk-input-sqlserver, returns Types.VARCHAR as JDBC type for these types.
+            return new StringColumnGetter(to,  getToType(option));
+
+        case "datetime":
+            ColumnGetter getter = super.newColumnGetter(con, task, column, option);
+            return new TimestampWithoutTimeZoneIncrementalHandler(getter);
+
+        default:
+            return super.newColumnGetter(con, task, column, option);
         }
     }
 

data/src/test/java/org/embulk/input/sqlserver/BasicTest.java

@@ -53,4 +53,13 @@ public class BasicTest
         assertThat(readSortedFile(out1), is(readResource("test_expected.csv")));
         assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
     }
+
+    @Test
+    public void testJTDS() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "jtds_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
 }

data/src/test/java/org/embulk/input/sqlserver/IncrementalTest.java

@@ -0,0 +1,149 @@
+package org.embulk.input.sqlserver;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.SQLServerInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.embulk.input.sqlserver.SQLServerTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class IncrementalTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/sqlserver/test/expect/incremental/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "sqlserver", SQLServerInputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = SQLServerTests.baseConfig();
+    }
+
+    @Test
+    public void testInt() throws Exception
+    {
+        // setup first rows
+        execute(readResource("int/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "int/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("int/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "int/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("int/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "int/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("int/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
+    }
+
+    @Test
+    public void testChar() throws Exception
+    {
+        // setup first rows
+        execute(readResource("char/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "char/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("char/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "char/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("char/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "char/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("char/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "char/expected_2.diff")));
+    }
+
+    @Test
+    public void testDateTime2() throws Exception
+    {
+        // setup first rows
+        execute(readResource("datetime2/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "datetime2/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("datetime2/expected_1.csv")));
+        // SQL Server datetime2 type is mapped to StringColumnGetter, not to TimestampWithoutTimeZoneIncrementalHandler, for compatibility.
+        // So a timestamp value in JSON will be like 'yyyy-MM-dd HH:mm:ss.SSSSSSS', not like 'yyyy-MM-ddTHH:mm:ss.SSSSSS'.
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "datetime2/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("datetime2/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "datetime2/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("datetime2/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "datetime2/expected_2.diff")));
+    }
+
+    @Test
+    public void testQueryWithPlaceholder() throws Exception
+    {
+        // setup first rows
+        execute(readResource("query/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "query/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("query/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "query/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("query/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "query/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("query/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "query/expected_2.diff")));
+    }
+
+    @Test
+    public void testQueryWithPlaceholderAndMultiColumns() throws Exception
+    {
+        // setup first rows
+        execute(readResource("query/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("query/multi_columns_expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("query/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "query/multi_columns_config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("query/multi_columns_expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "query/multi_columns_expected_2.diff")));
+    }
+}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/basic/jtds_config.yml

@@ -0,0 +1,3 @@
+table: TEST1
+order_by: 'ID'
+driver_type: jtds

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_1.yml

@@ -0,0 +1,3 @@
+table: CHAR_LOAD
+incremental: true
+incremental_columns: [NAME]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_2.yml

@@ -0,0 +1,4 @@
+table: CHAR_LOAD
+last_record: ['A4']
+incremental: true
+incremental_columns: [NAME]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.csv

@@ -0,0 +1,4 @@
+A1,first
+A2,first
+A3,first
+A4,first

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['A4']
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.csv

@@ -0,0 +1,2 @@
+A5,more_load
+A9,more_load

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['A9']
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/insert_more.sql

@@ -0,0 +1,6 @@
+
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A0', 'more_skip');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A4', 'more_skip');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A9', 'more_load');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A5', 'more_load');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/setup.sql

@@ -0,0 +1,11 @@
+DROP TABLE CHAR_LOAD
+CREATE TABLE CHAR_LOAD (
+    NAME CHAR(2) NOT NULL,
+    NOTE TEXT
+);
+
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A3', 'first');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A4', 'first');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A2', 'first');
+INSERT INTO CHAR_LOAD (NAME, NOTE) VALUES ('A1', 'first');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_1.yml

@@ -0,0 +1,4 @@
+table: DT_LOAD
+default_time_zone: +0300 # should be ignored
+incremental: true
+incremental_columns: [TIME]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_2.yml

@@ -0,0 +1,4 @@
+table: DT_LOAD
+last_record: ['2016-11-02 04:00:05.3330030']
+incremental: true
+incremental_columns: [TIME]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-02 01:00:01.0000000,first
+2016-11-02 02:00:02.0000000,first
+2016-11-02 03:00:03.0000000,first
+2016-11-02 04:00:04.0000000,first
+2016-11-02 04:00:05.1110010,first
+2016-11-02 04:00:05.2220020,first
+2016-11-02 04:00:05.3330030,first

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02 04:00:05.3330030']
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.csv

@@ -0,0 +1,3 @@
+2016-11-02 04:00:05.3330040,more_load
+2016-11-02 04:00:06.0000000,more_load
+2016-11-02 04:00:06.0000000,more_load

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: ['2016-11-02 04:00:06.0000000']
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/insert_more.sql

@@ -0,0 +1,8 @@
+
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:00', 'more_skip');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333000', 'more_skip');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333003', 'more_skip');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333004', 'more_load');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:06', 'more_load');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:06', 'more_load');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/setup.sql

@@ -0,0 +1,14 @@
+DROP TABLE DT_LOAD
+CREATE TABLE DT_LOAD (
+    TIME DATETIME2 NOT NULL,
+    NOTE TEXT
+);
+
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 01:00:01', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 02:00:02', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 03:00:03', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:04', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.111001', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.222002', 'first');
+INSERT INTO DT_LOAD (TIME, NOTE) VALUES ('2016-11-02 04:00:05.333003', 'first');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_1.yml

@@ -0,0 +1,3 @@
+table: INT_LOAD
+incremental: true
+incremental_columns: [NUM]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_2.yml

@@ -0,0 +1,4 @@
+table: INT_LOAD
+last_record: [4]
+incremental: true
+incremental_columns: [NUM]

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.csv

@@ -0,0 +1,4 @@
+1,first
+2,first
+3,first
+4,first

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.csv

@@ -0,0 +1,2 @@
+5,more_load
+9,more_load

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/insert_more.sql

@@ -0,0 +1,6 @@
+
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (0, 'more_skip');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (4, 'more_skip');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (9, 'more_load');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (5, 'more_load');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/setup.sql

@@ -0,0 +1,11 @@
+DROP TABLE INT_LOAD
+CREATE TABLE INT_LOAD (
+    NUM INT NOT NULL,
+    NOTE TEXT
+);
+
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (3, 'first');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (4, 'first');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (2, 'first');
+INSERT INTO INT_LOAD (NUM, NOTE) VALUES (1, 'first');
+

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_1.yml

@@ -0,0 +1,14 @@
+last_record: [0]
+incremental: true
+incremental_columns: [NUM]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    NUM,NOTE
+  FROM
+    QUERY_LOAD
+  WHERE
+    NUM IS NOT NULL
+    AND NUM > :NUM
+  ORDER BY
+    NUM ASC

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_2.yml

@@ -0,0 +1,14 @@
+last_record: [4]
+incremental: true
+incremental_columns: [NUM]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    NUM,NOTE
+  FROM
+    QUERY_LOAD
+  WHERE
+    NUM IS NOT NULL
+    AND NUM > :NUM
+  ORDER BY
+    NUM ASC

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.csv

@@ -0,0 +1,4 @@
+1,first
+2,first
+3,first
+4,first

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.csv

@@ -0,0 +1,2 @@
+5,more_load
+9,more_load

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/insert_more.sql

@@ -0,0 +1,5 @@
+
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (0, 100, 'more_skip');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (4, 104, 'more_skip');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (9, 109, 'more_load');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (5, 105, 'more_load');

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_1.yml

@@ -0,0 +1,16 @@
+last_record: [0,0]
+incremental: true
+incremental_columns: [NUM,NUM2]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    *
+  FROM
+    QUERY_LOAD
+  WHERE
+    NUM IS NOT NULL
+    AND NUM > :NUM
+    OR (NUM = :NUM AND NUM2 > :NUM2)
+  ORDER BY
+    NUM ASC,
+    NUM2 ASC

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_2.yml

@@ -0,0 +1,17 @@
+last_record: [4,104]
+incremental: true
+incremental_columns: [NUM,NUM2]
+use_raw_query_with_incremental: true
+query: |
+  SELECT
+    *
+  FROM
+    QUERY_LOAD
+  WHERE
+    NUM IS NOT NULL
+    AND NUM > :NUM
+    AND NUM2 IS NOT NULL
+    OR (NUM = :NUM AND NUM2 > :NUM2)
+  ORDER BY
+    NUM ASC,
+    NUM2 ASC

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.csv

@@ -0,0 +1,4 @@
+1,101,first
+2,102,first
+3,103,first
+4,104,first

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [4,104]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.csv

@@ -0,0 +1,2 @@
+5,105,more_load
+9,109,more_load

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.diff

@@ -0,0 +1,3 @@
+in:
+  last_record: [9,109]
+out: {}

data/src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/setup.sql

@@ -0,0 +1,11 @@
+DROP TABLE QUERY_LOAD
+CREATE TABLE QUERY_LOAD (
+    NUM INT NOT NULL,
+    NUM2 INT NOT NULL,
+    NOTE TEXT
+);
+
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (3, 103, 'first');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (4, 104, 'first');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (2, 102, 'first');
+INSERT INTO QUERY_LOAD (NUM, NUM2, NOTE) VALUES (1, 101, 'first');

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.10.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-23 00:00:00.000000000 Z
+date: 2019-09-06 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,19 +19,60 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.10.0.jar
-- classpath/embulk-input-sqlserver-0.10.0.jar
+- classpath/embulk-input-jdbc-0.10.1.jar
+- classpath/embulk-input-sqlserver-0.10.1.jar
 - classpath/jtds-1.3.1.jar
+- classpath/mssql-jdbc-7.2.2.jre8.jar
 - lib/embulk/input/sqlserver.rb
 - src/main/java/org/embulk/input/SQLServerInputPlugin.java
 - src/main/java/org/embulk/input/sqlserver/SQLServerInputConnection.java
 - src/main/java/org/embulk/input/sqlserver/getter/SQLServerColumnGetterFactory.java
 - src/test/java/org/embulk/input/sqlserver/BasicTest.java
+- src/test/java/org/embulk/input/sqlserver/IncrementalTest.java
 - src/test/java/org/embulk/input/sqlserver/SQLServerTests.java
+- src/test/resources/org/embulk/input/sqlserver/test/expect/basic/jtds_config.yml
 - src/test/resources/org/embulk/input/sqlserver/test/expect/basic/setup.sql
 - src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_config.yml
 - src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_expected.csv
 - src/test/resources/org/embulk/input/sqlserver/test/expect/basic/test_expected.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_1.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/config_2.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_1.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/expected_2.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/insert_more.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/char/setup.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_1.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/config_2.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_1.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/expected_2.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/insert_more.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/datetime2/setup.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_1.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/config_2.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_1.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/expected_2.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/insert_more.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/int/setup.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_1.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/config_2.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_1.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/expected_2.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/insert_more.sql
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_1.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_config_2.yml
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_1.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.csv
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/multi_columns_expected_2.diff
+- src/test/resources/org/embulk/input/sqlserver/test/expect/incremental/query/setup.sql
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0