embulk-input-sqlserver 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9446c62db7ee87a1b3aeaa0767078eac81925aa3
-  data.tar.gz: 542507e95e1b66d8f1cea1f69b8f0b75520071e5
+  metadata.gz: 2132f65d744523dabf21fea67e14e0fa41d7c2da
+  data.tar.gz: 1ca482dbf5a4b8990e9ec1ac6cc7e2ef50b965b8
 SHA512:
-  metadata.gz: 5b148f5fc32c79fb7c76686d8ad1716f877c03231c9b4350d4fa7d864ec0fe272cc4d4e9bfcd07a816b73ca0f602c699dae07fa81f82c2840fb214cc63b40894
-  data.tar.gz: 32db74d50e3bfdcec10394ba86c723c4e58bb944f0e443ea9952dbbc1b98c71f948a81f4a10893095dd3907b6a9a50dcee86080714c1a831cbe6715d7a5c83c6
+  metadata.gz: 8e136aaf3ec1160752c2d94de59f8d07553a5d27fe842b769c1742022eeea58a2061713aa93f178e2b79ed4e39194cac92399ae12b416ea4eba3688dd810c9b4
+  data.tar.gz: a519f7cf5dd52f0fd1c57cfe30b9dc0fbd4d0060cc1d14155e9c654624c2257775b42883ff4c9b637e8cb2b3f0fd2dabb272a3cd646e3ffae762ad956a9d50b9

data/README.md

@@ -9,7 +9,7 @@ SQL Server input plugins for Embulk loads records from SQL Server.
 
 ## Configuration
 
-- **driver_path**: path to the jar file of the SQL Server JDBC driver (string)
+- **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, open-source driver (jTDS driver) is used (string)
 - **host**: database host name (string, required if url is not set)
 - **port**: database port number (integer, default: 1433)
 - **integratedSecutiry**: whether to use integrated authentication or not. The `sqljdbc_auth.dll` must be located on Java library path if using integrated authentication. : (boolean, default: false)
@@ -19,20 +19,24 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 ```
 - **user**: database login user name (string, required if not using integrated authentication)
 - **password**: database login password (string, default: "")
-- **instance**: destination instance name (string, default: use the default instance)
+- **instance**: destination instance name. if instance is set, port option will be ignored (string, default: use the default instance)
 - **database**: destination database name (string, default: use the default database)
 - **url**: URL of the JDBC connection (string, optional)
 - If you write SQL directly,
   - **query**: SQL to run (string)
 - If **query** is not set,
   - **table**: destination table name (string, required)
-  - **select**: comma-separated list of columns to select (string, default: "*")
+  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
   - **where**: WHERE condition to filter the rows (string, default: no-condition)
-  - **order_by**: name of the column that rows are sorted by (string, default: not sorted)
+  - **order_by**: expression of ORDER BY to sort rows (e.g. `created_at DESC, id ASC`) (string, default: not sorted)
 - **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
 - **connect_timeout**: timeout for the driver to connect. 0 means the default of SQL Server (15 by default). (integer (seconds), default: 300)
+- **application_name**: application name used to identify a connection in profiling and logging tools. (string, default: "embulk-input-sqlserver")
 - **socket_timeout**: timeout for executing the query. 0 means no timeout. (integer (seconds), default: 1800)
 - **options**: extra JDBC properties (hash, default: {})
+- **incremental**: if true, enables incremental loading. See next section for details (boolean, default: false)
+- **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys)
+- **last_record**: values of the last record for incremental loading (array of objects, default: load all records)
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
   - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
@@ -45,6 +49,43 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 (string, value of default_timezone option is used by default)
 - **after_select**: if set, this SQL will be executed after the SELECT query in the same transaction.
 
+
+### Incremental loading
+
+Incremental loading uses monotonically increasing unique columns (such as IDENTITY column) to load records inserted (or updated) after last execution.
+
+First, if `incremental: true` is set, this plugin loads all records with additional ORDER BY. For example, if `incremental_columns: [updated_at, id]` option is set, query will be as following:
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+ORDER BY updated_at, id
+```
+
+When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
+
+At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+WHERE created_at > '2017-01-01 00:32:12' OR (created_at = '2017-01-01 00:32:12' AND id > 5291)
+ORDER BY updated_at, id
+```
+
+Then, it updates `last_record: ` so that next execution uses the updated last_record.
+
+**IMPORTANT**: If you set `incremental_columns: ` option, make sure that there is an index on the columns to avoid full table scan. For this example, following index should be created:
+
+```
+CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
+```
+
+Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an IDENTITY primary key. Currently, only strings and integers are supported as incremental_columns.
+
+
 ## Example
 
 ```yaml
@@ -59,6 +100,16 @@ in:
   table: my_table
   select: "col1, col2, col3"
   where: "col4 != 'a'"
+  order_by: "col1 DESC"
+```
+
+This configuration will generate following SQL:
+
+```
+SELECT col1, col2, col3
+FROM "my_table"
+WHERE col4 != 'a'
+ORDER BY col1 DESC
 ```
 
 If you need a complex SQL,

data/build.gradle

@@ -1,5 +1,6 @@
 dependencies {
     compile project(':embulk-input-jdbc')
+    compile 'net.sourceforge.jtds:jtds:1.3.1'
 
     testCompile project(':embulk-input-jdbc').sourceSets.test.output
 }

data/src/main/java/org/embulk/input/SQLServerInputPlugin.java

@@ -1,21 +1,26 @@
 package org.embulk.input;
 
 import java.sql.Connection;
-import java.sql.DriverManager;
+import java.sql.Driver;
 import java.sql.SQLException;
 import java.util.Properties;
+import javax.validation.constraints.Size;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
 import org.embulk.input.jdbc.JdbcInputConnection;
 import org.embulk.input.sqlserver.SQLServerInputConnection;
 
 import com.google.common.base.Optional;
+import static java.util.Locale.ENGLISH;
 
 public class SQLServerInputPlugin
     extends AbstractJdbcInputPlugin
 {
+    private static int DEFAULT_PORT = 1433;
+
     public interface SQLServerPluginTask
         extends PluginTask
     {
@@ -40,8 +45,8 @@ public class SQLServerInputPlugin
         public Optional<String> getDatabase();
 
         @Config("integratedSecurity")
-        @ConfigDefault("null")
-        public Optional<Boolean> getIntegratedSecurity();
+        @ConfigDefault("false")
+        public boolean getIntegratedSecurity();
 
         @Config("url")
         @ConfigDefault("null")
@@ -53,11 +58,16 @@ public class SQLServerInputPlugin
 
         @Config("password")
         @ConfigDefault("\"\"")
-        public Optional<String> getPassword();
+        public String getPassword();
 
         @Config("schema")
         @ConfigDefault("null")
         public Optional<String> getSchema();
+
+        @Config("application_name")
+        @ConfigDefault("\"embulk-input-sqlserver\"")
+        @Size(max=128)
+        public String getApplicationName();
     }
 
     @Override
@@ -66,81 +76,184 @@ public class SQLServerInputPlugin
         return SQLServerPluginTask.class;
     }
 
+    private static class UrlAndProperties
+    {
+        private final String url;
+        private final Properties properties;
+
+        public UrlAndProperties(String url, Properties properties)
+        {
+            this.url = url;
+            this.properties = properties;
+        }
+
+        public String getUrl()
+        {
+            return url;
+        }
+
+        public Properties getProperties()
+        {
+            return properties;
+        }
+    }
+
     @Override
     protected JdbcInputConnection newConnection(PluginTask task) throws SQLException
     {
         SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
 
-        String url;
-        if (sqlServerTask.getUrl().isPresent()) {
-            if (sqlServerTask.getHost().isPresent()
-                    || sqlServerTask.getInstance().isPresent()
-                    || sqlServerTask.getDatabase().isPresent()
-                    || sqlServerTask.getIntegratedSecurity().isPresent()) {
-                throw new IllegalArgumentException("'host', 'port', 'instance', 'database' and 'integratedSecurity' parameters are invalid if 'url' parameter is set.");
-            }
-            url = sqlServerTask.getUrl().get();
-        } else {
-            if (!sqlServerTask.getHost().isPresent()) {
-                throw new IllegalArgumentException("Field 'host' is not set.");
+        Driver driver;
+        boolean useJtdsDriver = false;
+        if (sqlServerTask.getDriverPath().isPresent()) {
+            addDriverJarToClasspath(sqlServerTask.getDriverPath().get());
+            try {
+                driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
             }
-            if (!sqlServerTask.getDatabase().isPresent()) {
-                throw new IllegalArgumentException("Field 'database' is not set.");
+            catch (Exception e) {
+                throw new ConfigException("Driver set at field 'driver_path' doesn't include Microsoft SQLServerDriver", e);
             }
-            StringBuilder urlBuilder = new StringBuilder();
-            if (sqlServerTask.getInstance().isPresent()) {
-                urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s",
-                        sqlServerTask.getHost().get(), sqlServerTask.getInstance().get()));
-            } else {
-                urlBuilder.append(String.format("jdbc:sqlserver://%s:%d",
-                        sqlServerTask.getHost().get(), sqlServerTask.getPort()));
+        }
+        else {
+            // prefer Microsoft SQLServerDriver if it is in classpath
+            try {
+                driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
             }
-            if (sqlServerTask.getDatabase().isPresent()) {
-                urlBuilder.append(";databaseName=" + sqlServerTask.getDatabase().get());
+            catch (Exception ex) {
+                logger.info("Using jTDS Driver");
+                driver = new net.sourceforge.jtds.jdbc.Driver();
+                useJtdsDriver = true;
             }
-            if (sqlServerTask.getIntegratedSecurity().isPresent() && sqlServerTask.getIntegratedSecurity().get()) {
-                urlBuilder.append(";integratedSecurity=" + sqlServerTask.getIntegratedSecurity().get());
-            } else {
-                if (!sqlServerTask.getUser().isPresent()) {
-                    throw new IllegalArgumentException("Field 'user' is not set.");
-                }
-                if (!sqlServerTask.getPassword().isPresent()) {
-                    throw new IllegalArgumentException("Field 'password' is not set.");
-                }
+        }
+
+        UrlAndProperties urlAndProps = buildUrlAndProperties(sqlServerTask, useJtdsDriver);
+
+        Properties props = urlAndProps.getProperties();
+        props.putAll(sqlServerTask.getOptions());
+
+        Connection con = driver.connect(urlAndProps.getUrl(), props);
+        try {
+            SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull());
+            con = null;
+            return c;
+        }
+        finally {
+            if (con != null) {
+                con.close();
             }
-            url = urlBuilder.toString();
         }
+    }
 
+    private UrlAndProperties buildUrlAndProperties(SQLServerPluginTask sqlServerTask, boolean useJtdsDriver)
+    {
         Properties props = new Properties();
+
+        // common properties
+
         if (sqlServerTask.getUser().isPresent()) {
             props.setProperty("user", sqlServerTask.getUser().get());
         }
-        if (sqlServerTask.getPassword().isPresent()) {
-            props.setProperty("password", sqlServerTask.getPassword().get());
+        props.setProperty("password", sqlServerTask.getPassword());
+
+        if (useJtdsDriver) {
+            // jTDS properties
+            props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
+            props.setProperty("socketTimeout", String.valueOf(sqlServerTask.getSocketTimeout())); // seconds
+
+            props.setProperty("appName", sqlServerTask.getApplicationName());
+
+            // TODO support more options as necessary
+            // List of properties: http://jtds.sourceforge.net/faq.html
         }
-        props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
-        props.putAll(sqlServerTask.getOptions());
+        else {
+            // SQLServerDriver properties
+            props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
 
-        if (sqlServerTask.getDriverPath().isPresent()) {
-            loadDriverJar(sqlServerTask.getDriverPath().get());
+            props.setProperty("applicationName", sqlServerTask.getApplicationName());
+
+            // TODO support more options as necessary
+            // List of properties: https://msdn.microsoft.com/en-us/library/ms378988(v=sql.110).aspx
         }
 
-        try {
-            Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-        } catch (Exception ex) {
-            throw new RuntimeException(ex);
+        // skip URL build if it's set
+        if (sqlServerTask.getUrl().isPresent()) {
+            if (sqlServerTask.getHost().isPresent()
+                    || sqlServerTask.getInstance().isPresent()
+                    || sqlServerTask.getDatabase().isPresent()
+                    || sqlServerTask.getIntegratedSecurity()) {
+                throw new ConfigException("'host', 'port', 'instance', 'database' and 'integratedSecurity' options are invalid if 'url' option is set.");
+            }
+
+            return new UrlAndProperties(sqlServerTask.getUrl().get(), props);
         }
 
-        Connection con = DriverManager.getConnection(url, props);
-        try {
-            SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull());
-            con = null;
-            return c;
-        } finally {
-            if (con != null) {
-                con.close();
+        // build URL
+        String url;
+
+        if (!sqlServerTask.getHost().isPresent()) {
+            throw new ConfigException("'host' option is required but not set.");
+        }
+
+        if (useJtdsDriver) {
+            // jTDS URL: host:port[/database] or host[/database][;instance=]
+            // host:port;instance= is allowed but port will be ignored? in this case.
+            if (sqlServerTask.getInstance().isPresent()) {
+                if (sqlServerTask.getPort() != DEFAULT_PORT) {
+                    logger.warn("'port: {}' option is ignored because instance option is set", sqlServerTask.getPort());
+                }
+                url = String.format(ENGLISH, "jdbc:jtds:sqlserver://%s", sqlServerTask.getHost().get());
+                props.setProperty("instance", sqlServerTask.getInstance().get());
+            }
+            else {
+                url = String.format(ENGLISH, "jdbc:jtds:sqlserver://%s:%d", sqlServerTask.getHost().get(), sqlServerTask.getPort());
+            }
+
+            // /database
+            if (sqlServerTask.getDatabase().isPresent()) {
+                url += "/" + sqlServerTask.getDatabase().get();
+            }
+
+            // integratedSecutiry is not supported, user + password is required
+            if (sqlServerTask.getIntegratedSecurity()) {
+                throw new ConfigException("'integratedSecutiry' option is not supported with jTDS driver. Set 'driver_path: /path/to/sqljdbc.jar' option if you want to use Microsoft SQLServerDriver.");
+            }
+
+            if (!sqlServerTask.getUser().isPresent()) {
+                throw new ConfigException("'user' option is required but not set.");
             }
         }
-    }
+        else {
+            // SQLServerDriver URL: host:port[;databaseName=] or host\instance[;databaseName=]
+            // host\instance:port[;databaseName] is allowed but \instance will be ignored in this case.
+            if (sqlServerTask.getInstance().isPresent()) {
+                if (sqlServerTask.getPort() != DEFAULT_PORT) {
+                    logger.warn("'port: {}' option is ignored because instance option is set", sqlServerTask.getPort());
+                }
+                url = String.format(ENGLISH, "jdbc:sqlserver://%s\\%s", sqlServerTask.getHost().get(), sqlServerTask.getInstance().get());
+            }
+            else {
+                url = String.format(ENGLISH, "jdbc:sqlserver://%s:%d", sqlServerTask.getHost().get(), sqlServerTask.getPort());
+            }
 
+            // ;databaseName=
+            if (sqlServerTask.getDatabase().isPresent()) {
+                props.setProperty("databaseName", sqlServerTask.getDatabase().get());
+            }
+
+            // integratedSecutiry or user + password is required
+            if (sqlServerTask.getIntegratedSecurity()) {
+                if (sqlServerTask.getUser().isPresent()) {
+                    throw new ConfigException("'user' options are invalid if 'integratedSecutiry' option is set.");
+                }
+                props.setProperty("integratedSecurity", "true");
+            }
+            else {
+                if (!sqlServerTask.getUser().isPresent()) {
+                    throw new ConfigException("'user' option is required but not set.");
+                }
+            }
+        }
+
+        return new UrlAndProperties(url, props);
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-08-26 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,11 +19,12 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
+- classpath/embulk-input-jdbc-0.7.3.jar
+- classpath/embulk-input-sqlserver-0.7.3.jar
+- classpath/jtds-1.3.1.jar
 - lib/embulk/input/sqlserver.rb
 - src/main/java/org/embulk/input/SQLServerInputPlugin.java
 - src/main/java/org/embulk/input/sqlserver/SQLServerInputConnection.java
-- classpath/embulk-input-jdbc-0.7.2.jar
-- classpath/embulk-input-sqlserver-0.7.2.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0
@@ -34,17 +35,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: JDBC input plugin for Embulk