embulk-output-sqlserver 0.7.11 → 0.7.12

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d8d8e2c0bbd2a1255d5825697f0c6f2e893d9753
4
- data.tar.gz: 14ee14826f9dea6d9d5549e7bdaa41bcb01d8943
3
+ metadata.gz: e3ed69cc6fcf1d2f3c438c92cef31529785ca59b
4
+ data.tar.gz: 9918999eda849dadc22bd7e49941b0f95e41fb48
5
5
  SHA512:
6
- metadata.gz: 913d8c0c5b02f0ef7488c3063b0e022b606774c6c0856d2a78670e2119a5b7708eb88efa1f1e47e4cbe9014ecf448eb0a867bc528d6ef76e67c4f493658a437c
7
- data.tar.gz: df036fa9b4a164a18238f56273ba578d962bc1f43d97037e9dc740aa0517fedaf624bef35443631e5df7b8afeb728712aa5d90473e38ac0ed73ec52f546261e0
6
+ metadata.gz: 2afd308537cba8f93859abaaf3e0b7058ac5fa160827062f2a76fbf491fd367224bf082c14a3ffc71639bf0b24da2fe793113d5181edbdc5d523e380ddef93b8
7
+ data.tar.gz: 6f56d6d312892061f81a463bb80cbdadd008c8613825d197befbba0c140e1038423a1bc0b27071d109ee23729aa3ac129e760079fe9079fe8f569aab49509144
data/README.md CHANGED
@@ -34,6 +34,8 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
34
34
  - **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
35
35
  - **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = T.foo + S.foo` (`T` means target table and `S` means source table). (string array, default: always overwrites with new values)
36
36
  - **insert_method**: see below
37
+ - **native_driver**: driver name when using `insert_method: native`. (string, default: `{SQL Server Native Client 11.0}`)
38
+ - **database_encoding**: database encoding when using `insert_method: native`. (string, default: `MS932`)
37
39
  - **batch_size**: size of a single batch insert (integer, default: 16777216)
38
40
  - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
39
41
  - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -74,7 +76,9 @@ insert_method supports three options.
74
76
  "normal" means normal insert (default). It requires SQL Server JDBC driver.
75
77
 
76
78
  "native" means bulk insert using native client. It is faster than "normal".
77
- It requires both SQL Server JDBC driver and SQL Server Native Client (11.0).
79
+ It requires both SQL Server JDBC driver and SQL Server Native Client (or newer version of Microsoft ODBC Driver).
80
+
81
+ If you embulk use on Linux, you needed to [install mssql-tools](https://docs.microsoft.com/en-us/sql/connect/odbc/linux-mac/installing-the-microsoft-odbc-driver-for-sql-server).
78
82
 
79
83
  ### Supported types
80
84
 
@@ -83,6 +83,14 @@ public class SQLServerOutputPlugin
83
83
  @Config("insert_method")
84
84
  @ConfigDefault("\"normal\"")
85
85
  public InsertMethod getInsertMethod();
86
+
87
+ @Config("native_driver")
88
+ @ConfigDefault("null")
89
+ public Optional<String> getNativeDriverName();
90
+
91
+ @Config("database_encoding")
92
+ @ConfigDefault("\"MS932\"")
93
+ public String getDatabaseEncoding();
86
94
  }
87
95
 
88
96
  private static class UrlAndProperties {
@@ -265,8 +273,10 @@ public class SQLServerOutputPlugin
265
273
  {
266
274
  SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
267
275
  if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
268
- return new NativeBatchInsert(sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
269
- sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword());
276
+ return new NativeBatchInsert(
277
+ sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
278
+ sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword(),
279
+ sqlServerTask.getNativeDriverName(), sqlServerTask.getDatabaseEncoding());
270
280
  }
271
281
  return new StandardBatchInsert(getConnector(task, true), mergeConfig);
272
282
  }
@@ -33,6 +33,8 @@ public class NativeBatchInsert implements BatchInsert
33
33
  private final String database;
34
34
  private final Optional<String> user;
35
35
  private final Optional<String> password;
36
+ private final Optional<String> nativeDriverName;
37
+ private final String databaseEncoding;
36
38
 
37
39
  private int batchWeight;
38
40
  private int batchRows;
@@ -45,7 +47,8 @@ public class NativeBatchInsert implements BatchInsert
45
47
 
46
48
 
47
49
  public NativeBatchInsert(String server, int port, Optional<String> instance,
48
- String database, Optional<String> user, Optional<String> password)
50
+ String database, Optional<String> user, Optional<String> password,
51
+ Optional<String> nativeDriverName, String databaseEncoding)
49
52
  {
50
53
  this.server = server;
51
54
  this.port = port;
@@ -53,7 +56,8 @@ public class NativeBatchInsert implements BatchInsert
53
56
  this.database = database;
54
57
  this.user = user;
55
58
  this.password = password;
56
-
59
+ this.nativeDriverName = nativeDriverName;
60
+ this.databaseEncoding = databaseEncoding;
57
61
  lastColumnIndex = 0;
58
62
  }
59
63
 
@@ -62,8 +66,7 @@ public class NativeBatchInsert implements BatchInsert
62
66
  public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
63
67
  {
64
68
  columnCount = insertSchema.getCount();
65
- client.open(server, port, instance, database, user, password, loadTable.getTableName());
66
-
69
+ client.open(server, port, instance, database, user, password, loadTable.getTableName(), nativeDriverName, databaseEncoding);
67
70
  formats = new DateFormat[insertSchema.getCount()];
68
71
  for (int i = 0; i < insertSchema.getCount(); i++) {
69
72
  JdbcColumn column = insertSchema.getColumn(i);
@@ -8,6 +8,7 @@ import java.util.HashMap;
8
8
  import java.util.Map;
9
9
 
10
10
  import jnr.ffi.LibraryLoader;
11
+ import jnr.ffi.Platform;
11
12
  import jnr.ffi.Pointer;
12
13
  import jnr.ffi.Runtime;
13
14
  import jnr.ffi.provider.jffi.ArrayMemoryIO;
@@ -33,35 +34,46 @@ public class NativeClientWrapper
33
34
 
34
35
  public NativeClientWrapper()
35
36
  {
37
+ Platform platform = Platform.getPlatform();
38
+ Platform.OS os = platform.getOS();
39
+ String odbcLibName;
40
+ String nativeClientLibName;
41
+ if (os == Platform.OS.WINDOWS) {
42
+ odbcLibName = "odbc32";
43
+ nativeClientLibName = "sqlncli11";
44
+ } else {
45
+ odbcLibName = "odbc";
46
+ nativeClientLibName = "msodbcsql";
47
+ }
36
48
  synchronized (NativeClientWrapper.class) {
37
49
  if (odbc == null) {
38
- logger.info("Loading SQL Server Native Client library (odbc32).");
50
+ logger.info(String.format("Loading SQL Server Native Client library (%s).", odbcLibName));
39
51
  try {
40
- odbc = LibraryLoader.create(ODBC.class).failImmediately().load("odbc32");
52
+ odbc = LibraryLoader.create(ODBC.class).failImmediately().load(odbcLibName);
41
53
  } catch (UnsatisfiedLinkError e) {
42
- throw new RuntimeException("odbc32.dll not found.", e);
54
+ throw new RuntimeException(platform.mapLibraryName(odbcLibName) + " not found.", e);
43
55
  }
44
56
  }
45
57
  if (client == null) {
46
- logger.info("Loading SQL Server Native Client library (sqlncli11).");
58
+ logger.info(String.format("Loading SQL Server Native Client library (%s).", nativeClientLibName));
47
59
  try {
48
- client = LibraryLoader.create(NativeClient.class).failImmediately().load("sqlncli11");
60
+ client = LibraryLoader.create(NativeClient.class).failImmediately().load(nativeClientLibName);
49
61
  } catch (UnsatisfiedLinkError e) {
50
- throw new RuntimeException("sqlncli11.dll not found.", e);
62
+ throw new RuntimeException(platform.mapLibraryName(nativeClientLibName) + " not found.", e);
51
63
  }
52
64
  }
53
65
  }
54
-
55
- charset = Charset.forName("MS932");
56
- wideCharset = Charset.forName("UTF-16LE");
57
66
  }
58
67
 
59
68
  public void open(String server, int port, Optional<String> instance,
60
69
  String database, Optional<String> user, Optional<String> password,
61
- String table)
70
+ String table, Optional<String> nativeDriverName,
71
+ String databaseEncoding)
62
72
  throws SQLException
63
73
  {
64
74
  // environment handle
75
+ charset = Charset.forName(databaseEncoding);
76
+ wideCharset = Charset.forName("UTF-16LE");
65
77
  Pointer envHandlePointer = createPointerPointer();
66
78
  checkSQLResult("SQLAllocHandle(SQL_HANDLE_ENV)", odbc.SQLAllocHandle(
67
79
  ODBC.SQL_HANDLE_ENV,
@@ -92,7 +104,11 @@ public class NativeClientWrapper
92
104
  ODBC.SQL_IS_INTEGER));
93
105
 
94
106
  StringBuilder connectionString = new StringBuilder();
95
- connectionString.append("Driver={SQL Server Native Client 11.0};");
107
+ if (nativeDriverName.isPresent()) {
108
+ connectionString.append(String.format("Driver=%s;", nativeDriverName.get()));
109
+ } else {
110
+ connectionString.append("Driver={SQL Server Native Client 11.0};");
111
+ }
96
112
  if (instance.isPresent()) {
97
113
  connectionString.append(String.format("Server=%s,%d\\%s;", server, port, instance.get()));
98
114
  } else {
@@ -152,14 +168,15 @@ public class NativeClientWrapper
152
168
  public int bindValue(int columnIndex, String value) throws SQLException
153
169
  {
154
170
  ByteBuffer bytes = charset.encode(value);
155
- Pointer pointer = prepareBuffer(columnIndex, bytes.remaining());
156
- pointer.put(0, bytes.array(), 0, bytes.remaining());
171
+ int size = bytes.remaining();
172
+ Pointer pointer = prepareBuffer(columnIndex, size);
173
+ pointer.put(0, bytes.array(), 0, size);
157
174
 
158
175
  checkBCPResult("bcp_bind", client.bcp_bind(
159
176
  odbcHandle,
160
177
  pointer,
161
178
  0,
162
- (int)pointer.size(),
179
+ size,
163
180
  null,
164
181
  0,
165
182
  NativeClient.SQLCHARACTER,
@@ -169,14 +186,15 @@ public class NativeClientWrapper
169
186
 
170
187
  public int bindValue(int columnIndex, boolean value) throws SQLException
171
188
  {
172
- Pointer pointer = prepareBuffer(columnIndex, 1);
189
+ int size = 1;
190
+ Pointer pointer = prepareBuffer(columnIndex, size);
173
191
  pointer.putByte(0, value ? (byte)1 : (byte)0);
174
192
 
175
193
  checkBCPResult("bcp_bind", client.bcp_bind(
176
194
  odbcHandle,
177
195
  pointer,
178
196
  0,
179
- (int)pointer.size(),
197
+ size,
180
198
  null,
181
199
  0,
182
200
  NativeClient.SQLBIT,
@@ -186,14 +204,15 @@ public class NativeClientWrapper
186
204
 
187
205
  public int bindValue(int columnIndex, byte value) throws SQLException
188
206
  {
189
- Pointer pointer = prepareBuffer(columnIndex, 1);
207
+ int size = 1;
208
+ Pointer pointer = prepareBuffer(columnIndex, size);
190
209
  pointer.putByte(0, value);
191
210
 
192
211
  checkBCPResult("bcp_bind", client.bcp_bind(
193
212
  odbcHandle,
194
213
  pointer,
195
214
  0,
196
- (int)pointer.size(),
215
+ size,
197
216
  null,
198
217
  0,
199
218
  NativeClient.SQLINT1,
@@ -203,14 +222,15 @@ public class NativeClientWrapper
203
222
 
204
223
  public int bindValue(int columnIndex, short value) throws SQLException
205
224
  {
206
- Pointer pointer = prepareBuffer(columnIndex, 2);
225
+ int size = 2;
226
+ Pointer pointer = prepareBuffer(columnIndex, size);
207
227
  pointer.putShort(0, value);
208
228
 
209
229
  checkBCPResult("bcp_bind", client.bcp_bind(
210
230
  odbcHandle,
211
231
  pointer,
212
232
  0,
213
- (int)pointer.size(),
233
+ size,
214
234
  null,
215
235
  0,
216
236
  NativeClient.SQLINT2,
@@ -220,14 +240,15 @@ public class NativeClientWrapper
220
240
 
221
241
  public int bindValue(int columnIndex, int value) throws SQLException
222
242
  {
223
- Pointer pointer = prepareBuffer(columnIndex, 4);
243
+ int size = 4;
244
+ Pointer pointer = prepareBuffer(columnIndex, size);
224
245
  pointer.putInt(0, value);
225
246
 
226
247
  checkBCPResult("bcp_bind", client.bcp_bind(
227
248
  odbcHandle,
228
249
  pointer,
229
250
  0,
230
- (int)pointer.size(),
251
+ size,
231
252
  null,
232
253
  0,
233
254
  NativeClient.SQLINT4,
@@ -237,14 +258,15 @@ public class NativeClientWrapper
237
258
 
238
259
  public int bindValue(int columnIndex, long value) throws SQLException
239
260
  {
240
- Pointer pointer = prepareBuffer(columnIndex, 8);
261
+ int size = 8;
262
+ Pointer pointer = prepareBuffer(columnIndex, size);
241
263
  pointer.putLongLong(0, value);
242
264
 
243
265
  checkBCPResult("bcp_bind", client.bcp_bind(
244
266
  odbcHandle,
245
267
  pointer,
246
268
  0,
247
- (int)pointer.size(),
269
+ size,
248
270
  null,
249
271
  0,
250
272
  NativeClient.SQLINT8,
@@ -254,14 +276,15 @@ public class NativeClientWrapper
254
276
 
255
277
  public int bindValue(int columnIndex, float value) throws SQLException
256
278
  {
257
- Pointer pointer = prepareBuffer(columnIndex, 4);
279
+ int size = 4;
280
+ Pointer pointer = prepareBuffer(columnIndex, size);
258
281
  pointer.putFloat(0, value);
259
282
 
260
283
  checkBCPResult("bcp_bind", client.bcp_bind(
261
284
  odbcHandle,
262
285
  pointer,
263
286
  0,
264
- (int)pointer.size(),
287
+ size,
265
288
  null,
266
289
  0,
267
290
  NativeClient.SQLFLT4,
@@ -271,14 +294,15 @@ public class NativeClientWrapper
271
294
 
272
295
  public int bindValue(int columnIndex, double value) throws SQLException
273
296
  {
274
- Pointer pointer = prepareBuffer(columnIndex, 8);
297
+ int size = 8;
298
+ Pointer pointer = prepareBuffer(columnIndex, size);
275
299
  pointer.putDouble(0, value);
276
300
 
277
301
  checkBCPResult("bcp_bind", client.bcp_bind(
278
302
  odbcHandle,
279
303
  pointer,
280
304
  0,
281
- (int)pointer.size(),
305
+ size,
282
306
  null,
283
307
  0,
284
308
  NativeClient.SQLFLT8,
@@ -2,3 +2,4 @@ ID:string,TINYINT_ITEM:long,SMALLINT_ITEM:long,INT_ITEM:long,BIGINT_ITEM:long,BI
2
2
  A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900
3
3
  A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000,-999999999999000000,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-31 00:00:00.0 +0900,2017-12-31 23:59:59.997 +0900,2018-12-31 23:59:59.9999999 +0900,2019-12-31 23:59:59.99 +0900,2020-12-31 23:59:59.0 +0900,1970-01-01 23:59:59.9999999 +0900,1970-01-01 23:59:59.99 +0900
4
4
  A003,,,,,,,,,,,,,,,,,,,,,,,,
5
+ A004,9,9,9,9,0,9,9,9,9,9,9,x,x,x,X,X,X,2016-1-1 00:00:00.0 +0900,2017-1-1 1:2:3.123 +0900,2018-1-1 1:2:3.1234567 +0900,2019-1-1 1:2:3.12 +0900,2020-1-1 1:2:3.0 +0900,1970-01-01 3:4:5.1234567 +0900,1970-01-01 6:7:8.12 +0900
@@ -1,3 +1,4 @@
1
1
  A001,0,1234,123456,12345678901,0,1.23,3.456,12.34,123.4567,0.1234567,0.12345678901234,a,b,c,A,B,C,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000
2
2
  A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9999000000.0,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき,かきくけ,かきくけこ,2016-12-30 15:00:00.0000000,2017-12-31 14:59:59.9970000,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.9900000,2020-12-31 14:59:59.0000000,1970-01-01 14:59:59.9999990,1970-01-01 14:59:59.9900000
3
3
  A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
4
+ A004,9,9,9,9,0,9,9,9,9,9.0,9.0,x,x,x,X,X,X,2015-12-31 15:00:00.0000000,2016-12-31 16:02:03.1230000,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.1200000,2019-12-31 16:02:03.0000000,1969-12-31 18:04:05.1234560,1969-12-31 21:07:08.1200000
@@ -2,3 +2,4 @@
2
2
  A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a ,b,c,A ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
3
3
  A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99
4
4
  A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
5
+ A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x ,x,x,X ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
@@ -1,3 +1,4 @@
1
1
  A001,0,1234,123456,12345678901,0,1.23,3.456,12.3400,123.4567,0.1234567,0.12345678901234,a ,b,c,A ,B,C,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
2
2
  A002,255,-32768,-2147483648,-9223372036854775808,1,-9999999999.99,-99.999,-214748.3648,-922337203685477.5808,-9.9989996E+9,-9.9999999999900006E+17,あい,あいうえ,あいうえお,かき ,かきくけ,かきくけこ,2016-12-30,2017-12-31 14:59:59.997,2018-12-31 14:59:59.9999990,2019-12-31 14:59:59.99,2020-12-31 15:00:00,14:59:59.9999990,14:59:59.99
3
3
  A003,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL
4
+ A004,9,9,9,9,0,9.00,9.000,9.0000,9.0000,9.0,9.0,x ,x,x,X ,X,X,2015-12-31,2016-12-31 16:02:03.123,2017-12-31 16:02:03.1234560,2018-12-31 16:02:03.12,2019-12-31 16:02:00,18:04:05.1234560,21:07:08.12
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-sqlserver
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.11
4
+ version: 0.7.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-08-16 00:00:00.000000000 Z
11
+ date: 2017-11-24 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-output-jdbc-0.7.11.jar
23
- - classpath/embulk-output-sqlserver-0.7.11.jar
22
+ - classpath/embulk-output-jdbc-0.7.12.jar
23
+ - classpath/embulk-output-sqlserver-0.7.12.jar
24
24
  - classpath/jtds-1.3.1.jar
25
25
  - lib/embulk/output/sqlserver.rb
26
26
  - src/main/java/org/embulk/output/SQLServerOutputPlugin.java