embulk-input-sqlserver 0.7.2 → 0.7.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9446c62db7ee87a1b3aeaa0767078eac81925aa3
4
- data.tar.gz: 542507e95e1b66d8f1cea1f69b8f0b75520071e5
3
+ metadata.gz: 2132f65d744523dabf21fea67e14e0fa41d7c2da
4
+ data.tar.gz: 1ca482dbf5a4b8990e9ec1ac6cc7e2ef50b965b8
5
5
  SHA512:
6
- metadata.gz: 5b148f5fc32c79fb7c76686d8ad1716f877c03231c9b4350d4fa7d864ec0fe272cc4d4e9bfcd07a816b73ca0f602c699dae07fa81f82c2840fb214cc63b40894
7
- data.tar.gz: 32db74d50e3bfdcec10394ba86c723c4e58bb944f0e443ea9952dbbc1b98c71f948a81f4a10893095dd3907b6a9a50dcee86080714c1a831cbe6715d7a5c83c6
6
+ metadata.gz: 8e136aaf3ec1160752c2d94de59f8d07553a5d27fe842b769c1742022eeea58a2061713aa93f178e2b79ed4e39194cac92399ae12b416ea4eba3688dd810c9b4
7
+ data.tar.gz: a519f7cf5dd52f0fd1c57cfe30b9dc0fbd4d0060cc1d14155e9c654624c2257775b42883ff4c9b637e8cb2b3f0fd2dabb272a3cd646e3ffae762ad956a9d50b9
data/README.md CHANGED
@@ -9,7 +9,7 @@ SQL Server input plugins for Embulk loads records from SQL Server.
9
9
 
10
10
  ## Configuration
11
11
 
12
- - **driver_path**: path to the jar file of the SQL Server JDBC driver (string)
12
+ - **driver_path**: path to the jar file of Microsoft SQL Server JDBC driver. If not set, open-source driver (jTDS driver) is used (string)
13
13
  - **host**: database host name (string, required if url is not set)
14
14
  - **port**: database port number (integer, default: 1433)
15
15
  - **integratedSecutiry**: whether to use integrated authentication or not. The `sqljdbc_auth.dll` must be located on Java library path if using integrated authentication. : (boolean, default: false)
@@ -19,20 +19,24 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
19
19
  ```
20
20
  - **user**: database login user name (string, required if not using integrated authentication)
21
21
  - **password**: database login password (string, default: "")
22
- - **instance**: destination instance name (string, default: use the default instance)
22
+ - **instance**: destination instance name. if instance is set, port option will be ignored (string, default: use the default instance)
23
23
  - **database**: destination database name (string, default: use the default database)
24
24
  - **url**: URL of the JDBC connection (string, optional)
25
25
  - If you write SQL directly,
26
26
  - **query**: SQL to run (string)
27
27
  - If **query** is not set,
28
28
  - **table**: destination table name (string, required)
29
- - **select**: comma-separated list of columns to select (string, default: "*")
29
+ - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
30
30
  - **where**: WHERE condition to filter the rows (string, default: no-condition)
31
- - **order_by**: name of the column that rows are sorted by (string, default: not sorted)
31
+ - **order_by**: expression of ORDER BY to sort rows (e.g. `created_at DESC, id ASC`) (string, default: not sorted)
32
32
  - **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
33
33
  - **connect_timeout**: timeout for the driver to connect. 0 means the default of SQL Server (15 by default). (integer (seconds), default: 300)
34
+ - **application_name**: application name used to identify a connection in profiling and logging tools. (string, default: "embulk-input-sqlserver")
34
35
  - **socket_timeout**: timeout for executing the query. 0 means no timeout. (integer (seconds), default: 1800)
35
36
  - **options**: extra JDBC properties (hash, default: {})
37
+ - **incremental**: if true, enables incremental loading. See next section for details (boolean, default: false)
38
+ - **incremental_columns**: column names for incremental loading (array of strings, default: use primary keys)
39
+ - **last_record**: values of the last record for incremental loading (array of objects, default: load all records)
36
40
  - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
37
41
  - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
38
42
  - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`.
@@ -45,6 +49,43 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
45
49
  (string, value of default_timezone option is used by default)
46
50
  - **after_select**: if set, this SQL will be executed after the SELECT query in the same transaction.
47
51
 
52
+
53
+ ### Incremental loading
54
+
55
+ Incremental loading uses monotonically increasing unique columns (such as IDENTITY column) to load records inserted (or updated) after last execution.
56
+
57
+ First, if `incremental: true` is set, this plugin loads all records with additional ORDER BY. For example, if `incremental_columns: [updated_at, id]` option is set, query will be as following:
58
+
59
+ ```
60
+ SELECT * FROM (
61
+ ...original query is here...
62
+ )
63
+ ORDER BY updated_at, id
64
+ ```
65
+
66
+ When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
67
+
68
+ At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
69
+
70
+ ```
71
+ SELECT * FROM (
72
+ ...original query is here...
73
+ )
74
+ WHERE created_at > '2017-01-01 00:32:12' OR (created_at = '2017-01-01 00:32:12' AND id > 5291)
75
+ ORDER BY updated_at, id
76
+ ```
77
+
78
+ Then, it updates `last_record: ` so that next execution uses the updated last_record.
79
+
80
+ **IMPORTANT**: If you set `incremental_columns: ` option, make sure that there is an index on the columns to avoid full table scan. For this example, following index should be created:
81
+
82
+ ```
83
+ CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
84
+ ```
85
+
86
+ Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an IDENTITY primary key. Currently, only strings and integers are supported as incremental_columns.
87
+
88
+
48
89
  ## Example
49
90
 
50
91
  ```yaml
@@ -59,6 +100,16 @@ in:
59
100
  table: my_table
60
101
  select: "col1, col2, col3"
61
102
  where: "col4 != 'a'"
103
+ order_by: "col1 DESC"
104
+ ```
105
+
106
+ This configuration will generate following SQL:
107
+
108
+ ```
109
+ SELECT col1, col2, col3
110
+ FROM "my_table"
111
+ WHERE col4 != 'a'
112
+ ORDER BY col1 DESC
62
113
  ```
63
114
 
64
115
  If you need a complex SQL,
data/build.gradle CHANGED
@@ -1,5 +1,6 @@
1
1
  dependencies {
2
2
  compile project(':embulk-input-jdbc')
3
+ compile 'net.sourceforge.jtds:jtds:1.3.1'
3
4
 
4
5
  testCompile project(':embulk-input-jdbc').sourceSets.test.output
5
6
  }
Binary file
@@ -1,21 +1,26 @@
1
1
  package org.embulk.input;
2
2
 
3
3
  import java.sql.Connection;
4
- import java.sql.DriverManager;
4
+ import java.sql.Driver;
5
5
  import java.sql.SQLException;
6
6
  import java.util.Properties;
7
+ import javax.validation.constraints.Size;
7
8
 
8
9
  import org.embulk.config.Config;
9
10
  import org.embulk.config.ConfigDefault;
11
+ import org.embulk.config.ConfigException;
10
12
  import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
11
13
  import org.embulk.input.jdbc.JdbcInputConnection;
12
14
  import org.embulk.input.sqlserver.SQLServerInputConnection;
13
15
 
14
16
  import com.google.common.base.Optional;
17
+ import static java.util.Locale.ENGLISH;
15
18
 
16
19
  public class SQLServerInputPlugin
17
20
  extends AbstractJdbcInputPlugin
18
21
  {
22
+ private static int DEFAULT_PORT = 1433;
23
+
19
24
  public interface SQLServerPluginTask
20
25
  extends PluginTask
21
26
  {
@@ -40,8 +45,8 @@ public class SQLServerInputPlugin
40
45
  public Optional<String> getDatabase();
41
46
 
42
47
  @Config("integratedSecurity")
43
- @ConfigDefault("null")
44
- public Optional<Boolean> getIntegratedSecurity();
48
+ @ConfigDefault("false")
49
+ public boolean getIntegratedSecurity();
45
50
 
46
51
  @Config("url")
47
52
  @ConfigDefault("null")
@@ -53,11 +58,16 @@ public class SQLServerInputPlugin
53
58
 
54
59
  @Config("password")
55
60
  @ConfigDefault("\"\"")
56
- public Optional<String> getPassword();
61
+ public String getPassword();
57
62
 
58
63
  @Config("schema")
59
64
  @ConfigDefault("null")
60
65
  public Optional<String> getSchema();
66
+
67
+ @Config("application_name")
68
+ @ConfigDefault("\"embulk-input-sqlserver\"")
69
+ @Size(max=128)
70
+ public String getApplicationName();
61
71
  }
62
72
 
63
73
  @Override
@@ -66,81 +76,184 @@ public class SQLServerInputPlugin
66
76
  return SQLServerPluginTask.class;
67
77
  }
68
78
 
79
+ private static class UrlAndProperties
80
+ {
81
+ private final String url;
82
+ private final Properties properties;
83
+
84
+ public UrlAndProperties(String url, Properties properties)
85
+ {
86
+ this.url = url;
87
+ this.properties = properties;
88
+ }
89
+
90
+ public String getUrl()
91
+ {
92
+ return url;
93
+ }
94
+
95
+ public Properties getProperties()
96
+ {
97
+ return properties;
98
+ }
99
+ }
100
+
69
101
  @Override
70
102
  protected JdbcInputConnection newConnection(PluginTask task) throws SQLException
71
103
  {
72
104
  SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
73
105
 
74
- String url;
75
- if (sqlServerTask.getUrl().isPresent()) {
76
- if (sqlServerTask.getHost().isPresent()
77
- || sqlServerTask.getInstance().isPresent()
78
- || sqlServerTask.getDatabase().isPresent()
79
- || sqlServerTask.getIntegratedSecurity().isPresent()) {
80
- throw new IllegalArgumentException("'host', 'port', 'instance', 'database' and 'integratedSecurity' parameters are invalid if 'url' parameter is set.");
81
- }
82
- url = sqlServerTask.getUrl().get();
83
- } else {
84
- if (!sqlServerTask.getHost().isPresent()) {
85
- throw new IllegalArgumentException("Field 'host' is not set.");
106
+ Driver driver;
107
+ boolean useJtdsDriver = false;
108
+ if (sqlServerTask.getDriverPath().isPresent()) {
109
+ addDriverJarToClasspath(sqlServerTask.getDriverPath().get());
110
+ try {
111
+ driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
86
112
  }
87
- if (!sqlServerTask.getDatabase().isPresent()) {
88
- throw new IllegalArgumentException("Field 'database' is not set.");
113
+ catch (Exception e) {
114
+ throw new ConfigException("Driver set at field 'driver_path' doesn't include Microsoft SQLServerDriver", e);
89
115
  }
90
- StringBuilder urlBuilder = new StringBuilder();
91
- if (sqlServerTask.getInstance().isPresent()) {
92
- urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s",
93
- sqlServerTask.getHost().get(), sqlServerTask.getInstance().get()));
94
- } else {
95
- urlBuilder.append(String.format("jdbc:sqlserver://%s:%d",
96
- sqlServerTask.getHost().get(), sqlServerTask.getPort()));
116
+ }
117
+ else {
118
+ // prefer Microsoft SQLServerDriver if it is in classpath
119
+ try {
120
+ driver = (Driver) Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver").newInstance();
97
121
  }
98
- if (sqlServerTask.getDatabase().isPresent()) {
99
- urlBuilder.append(";databaseName=" + sqlServerTask.getDatabase().get());
122
+ catch (Exception ex) {
123
+ logger.info("Using jTDS Driver");
124
+ driver = new net.sourceforge.jtds.jdbc.Driver();
125
+ useJtdsDriver = true;
100
126
  }
101
- if (sqlServerTask.getIntegratedSecurity().isPresent() && sqlServerTask.getIntegratedSecurity().get()) {
102
- urlBuilder.append(";integratedSecurity=" + sqlServerTask.getIntegratedSecurity().get());
103
- } else {
104
- if (!sqlServerTask.getUser().isPresent()) {
105
- throw new IllegalArgumentException("Field 'user' is not set.");
106
- }
107
- if (!sqlServerTask.getPassword().isPresent()) {
108
- throw new IllegalArgumentException("Field 'password' is not set.");
109
- }
127
+ }
128
+
129
+ UrlAndProperties urlAndProps = buildUrlAndProperties(sqlServerTask, useJtdsDriver);
130
+
131
+ Properties props = urlAndProps.getProperties();
132
+ props.putAll(sqlServerTask.getOptions());
133
+
134
+ Connection con = driver.connect(urlAndProps.getUrl(), props);
135
+ try {
136
+ SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull());
137
+ con = null;
138
+ return c;
139
+ }
140
+ finally {
141
+ if (con != null) {
142
+ con.close();
110
143
  }
111
- url = urlBuilder.toString();
112
144
  }
145
+ }
113
146
 
147
+ private UrlAndProperties buildUrlAndProperties(SQLServerPluginTask sqlServerTask, boolean useJtdsDriver)
148
+ {
114
149
  Properties props = new Properties();
150
+
151
+ // common properties
152
+
115
153
  if (sqlServerTask.getUser().isPresent()) {
116
154
  props.setProperty("user", sqlServerTask.getUser().get());
117
155
  }
118
- if (sqlServerTask.getPassword().isPresent()) {
119
- props.setProperty("password", sqlServerTask.getPassword().get());
156
+ props.setProperty("password", sqlServerTask.getPassword());
157
+
158
+ if (useJtdsDriver) {
159
+ // jTDS properties
160
+ props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
161
+ props.setProperty("socketTimeout", String.valueOf(sqlServerTask.getSocketTimeout())); // seconds
162
+
163
+ props.setProperty("appName", sqlServerTask.getApplicationName());
164
+
165
+ // TODO support more options as necessary
166
+ // List of properties: http://jtds.sourceforge.net/faq.html
120
167
  }
121
- props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
122
- props.putAll(sqlServerTask.getOptions());
168
+ else {
169
+ // SQLServerDriver properties
170
+ props.setProperty("loginTimeout", String.valueOf(sqlServerTask.getConnectTimeout())); // seconds
123
171
 
124
- if (sqlServerTask.getDriverPath().isPresent()) {
125
- loadDriverJar(sqlServerTask.getDriverPath().get());
172
+ props.setProperty("applicationName", sqlServerTask.getApplicationName());
173
+
174
+ // TODO support more options as necessary
175
+ // List of properties: https://msdn.microsoft.com/en-us/library/ms378988(v=sql.110).aspx
126
176
  }
127
177
 
128
- try {
129
- Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
130
- } catch (Exception ex) {
131
- throw new RuntimeException(ex);
178
+ // skip URL build if it's set
179
+ if (sqlServerTask.getUrl().isPresent()) {
180
+ if (sqlServerTask.getHost().isPresent()
181
+ || sqlServerTask.getInstance().isPresent()
182
+ || sqlServerTask.getDatabase().isPresent()
183
+ || sqlServerTask.getIntegratedSecurity()) {
184
+ throw new ConfigException("'host', 'port', 'instance', 'database' and 'integratedSecurity' options are invalid if 'url' option is set.");
185
+ }
186
+
187
+ return new UrlAndProperties(sqlServerTask.getUrl().get(), props);
132
188
  }
133
189
 
134
- Connection con = DriverManager.getConnection(url, props);
135
- try {
136
- SQLServerInputConnection c = new SQLServerInputConnection(con, sqlServerTask.getSchema().orNull());
137
- con = null;
138
- return c;
139
- } finally {
140
- if (con != null) {
141
- con.close();
190
+ // build URL
191
+ String url;
192
+
193
+ if (!sqlServerTask.getHost().isPresent()) {
194
+ throw new ConfigException("'host' option is required but not set.");
195
+ }
196
+
197
+ if (useJtdsDriver) {
198
+ // jTDS URL: host:port[/database] or host[/database][;instance=]
199
+ // host:port;instance= is allowed but port will be ignored? in this case.
200
+ if (sqlServerTask.getInstance().isPresent()) {
201
+ if (sqlServerTask.getPort() != DEFAULT_PORT) {
202
+ logger.warn("'port: {}' option is ignored because instance option is set", sqlServerTask.getPort());
203
+ }
204
+ url = String.format(ENGLISH, "jdbc:jtds:sqlserver://%s", sqlServerTask.getHost().get());
205
+ props.setProperty("instance", sqlServerTask.getInstance().get());
206
+ }
207
+ else {
208
+ url = String.format(ENGLISH, "jdbc:jtds:sqlserver://%s:%d", sqlServerTask.getHost().get(), sqlServerTask.getPort());
209
+ }
210
+
211
+ // /database
212
+ if (sqlServerTask.getDatabase().isPresent()) {
213
+ url += "/" + sqlServerTask.getDatabase().get();
214
+ }
215
+
216
+ // integratedSecutiry is not supported, user + password is required
217
+ if (sqlServerTask.getIntegratedSecurity()) {
218
+ throw new ConfigException("'integratedSecutiry' option is not supported with jTDS driver. Set 'driver_path: /path/to/sqljdbc.jar' option if you want to use Microsoft SQLServerDriver.");
219
+ }
220
+
221
+ if (!sqlServerTask.getUser().isPresent()) {
222
+ throw new ConfigException("'user' option is required but not set.");
142
223
  }
143
224
  }
144
- }
225
+ else {
226
+ // SQLServerDriver URL: host:port[;databaseName=] or host\instance[;databaseName=]
227
+ // host\instance:port[;databaseName] is allowed but \instance will be ignored in this case.
228
+ if (sqlServerTask.getInstance().isPresent()) {
229
+ if (sqlServerTask.getPort() != DEFAULT_PORT) {
230
+ logger.warn("'port: {}' option is ignored because instance option is set", sqlServerTask.getPort());
231
+ }
232
+ url = String.format(ENGLISH, "jdbc:sqlserver://%s\\%s", sqlServerTask.getHost().get(), sqlServerTask.getInstance().get());
233
+ }
234
+ else {
235
+ url = String.format(ENGLISH, "jdbc:sqlserver://%s:%d", sqlServerTask.getHost().get(), sqlServerTask.getPort());
236
+ }
145
237
 
238
+ // ;databaseName=
239
+ if (sqlServerTask.getDatabase().isPresent()) {
240
+ props.setProperty("databaseName", sqlServerTask.getDatabase().get());
241
+ }
242
+
243
+ // integratedSecutiry or user + password is required
244
+ if (sqlServerTask.getIntegratedSecurity()) {
245
+ if (sqlServerTask.getUser().isPresent()) {
246
+ throw new ConfigException("'user' options are invalid if 'integratedSecutiry' option is set.");
247
+ }
248
+ props.setProperty("integratedSecurity", "true");
249
+ }
250
+ else {
251
+ if (!sqlServerTask.getUser().isPresent()) {
252
+ throw new ConfigException("'user' option is required but not set.");
253
+ }
254
+ }
255
+ }
256
+
257
+ return new UrlAndProperties(url, props);
258
+ }
146
259
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-sqlserver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 0.7.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-24 00:00:00.000000000 Z
11
+ date: 2016-08-26 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Selects records from a table.
14
14
  email:
@@ -19,11 +19,12 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
+ - classpath/embulk-input-jdbc-0.7.3.jar
23
+ - classpath/embulk-input-sqlserver-0.7.3.jar
24
+ - classpath/jtds-1.3.1.jar
22
25
  - lib/embulk/input/sqlserver.rb
23
26
  - src/main/java/org/embulk/input/SQLServerInputPlugin.java
24
27
  - src/main/java/org/embulk/input/sqlserver/SQLServerInputConnection.java
25
- - classpath/embulk-input-jdbc-0.7.2.jar
26
- - classpath/embulk-input-sqlserver-0.7.2.jar
27
28
  homepage: https://github.com/embulk/embulk-input-jdbc
28
29
  licenses:
29
30
  - Apache 2.0
@@ -34,17 +35,17 @@ require_paths:
34
35
  - lib
35
36
  required_ruby_version: !ruby/object:Gem::Requirement
36
37
  requirements:
37
- - - '>='
38
+ - - ">="
38
39
  - !ruby/object:Gem::Version
39
40
  version: '0'
40
41
  required_rubygems_version: !ruby/object:Gem::Requirement
41
42
  requirements:
42
- - - '>='
43
+ - - ">="
43
44
  - !ruby/object:Gem::Version
44
45
  version: '0'
45
46
  requirements: []
46
47
  rubyforge_project:
47
- rubygems_version: 2.1.9
48
+ rubygems_version: 2.4.8
48
49
  signing_key:
49
50
  specification_version: 4
50
51
  summary: JDBC input plugin for Embulk
Binary file