embulk-output-sqlserver 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -3
  3. data/classpath/{embulk-output-jdbc-0.5.0.jar → embulk-output-jdbc-0.5.1.jar} +0 -0
  4. data/classpath/embulk-output-sqlserver-0.5.1.jar +0 -0
  5. data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java +22 -4
  6. data/src/main/java/org/embulk/output/sqlserver/InsertMethod.java +32 -0
  7. data/src/main/java/org/embulk/output/sqlserver/NativeBatchInsert.java +247 -0
  8. data/src/main/java/org/embulk/output/sqlserver/SmallDateTimeFormat.java +31 -0
  9. data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClient.java +33 -0
  10. data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClientWrapper.java +442 -0
  11. data/src/main/java/org/embulk/output/sqlserver/nativeclient/ODBC.java +47 -0
  12. data/src/test/java/org/embulk/output/sqlserver/SQLServerOutputPluginTest.java +174 -10
  13. data/src/test/resources/sqlserver/data/test2/test2.csv +2 -0
  14. data/src/test/resources/sqlserver/data/test3/test3.csv +2 -0
  15. data/src/test/resources/sqlserver/data/test4/test4.csv +2 -0
  16. data/src/test/resources/sqlserver/data/test5/test5.csv +2 -0
  17. data/src/test/resources/sqlserver/yml/test-native-date.yml +26 -0
  18. data/src/test/resources/sqlserver/yml/test-native-decimal.yml +25 -0
  19. data/src/test/resources/sqlserver/yml/test-native-integer.yml +24 -0
  20. data/src/test/resources/sqlserver/yml/test-native-string.yml +25 -0
  21. data/src/test/resources/sqlserver/yml/test-native.yml +44 -0
  22. metadata +19 -4
  23. data/classpath/embulk-output-sqlserver-0.5.0.jar +0 -0
@@ -0,0 +1,442 @@
1
+ package org.embulk.output.sqlserver.nativeclient;
2
+
3
+ import java.nio.ByteBuffer;
4
+ import java.nio.CharBuffer;
5
+ import java.nio.charset.Charset;
6
+ import java.sql.SQLException;
7
+ import java.util.HashMap;
8
+ import java.util.Map;
9
+
10
+ import jnr.ffi.LibraryLoader;
11
+ import jnr.ffi.Pointer;
12
+ import jnr.ffi.Runtime;
13
+ import jnr.ffi.provider.jffi.ArrayMemoryIO;
14
+
15
+ import org.embulk.spi.Exec;
16
+ import org.slf4j.Logger;
17
+
18
+ import com.google.common.base.Optional;
19
+
20
+ public class NativeClientWrapper
21
+ {
22
+ private static ODBC odbc;
23
+ private static NativeClient client;
24
+
25
+ private final Logger logger = Exec.getLogger(getClass());
26
+
27
+ private Charset charset;
28
+ private Charset wideCharset;
29
+ private Pointer envHandle;
30
+ private Pointer odbcHandle;
31
+
32
+ private Map<Integer, Pointer> boundPointers = new HashMap<Integer, Pointer>();
33
+
34
+ public NativeClientWrapper()
35
+ {
36
+ synchronized (NativeClientWrapper.class) {
37
+ if (odbc == null) {
38
+ logger.info("Loading SQL Server Native Client library (odbc32).");
39
+ try {
40
+ odbc = LibraryLoader.create(ODBC.class).failImmediately().load("odbc32");
41
+ } catch (UnsatisfiedLinkError e) {
42
+ throw new RuntimeException("odbc32.dll not found.", e);
43
+ }
44
+ }
45
+ if (client == null) {
46
+ logger.info("Loading SQL Server Native Client library (sqlncli11).");
47
+ try {
48
+ client = LibraryLoader.create(NativeClient.class).failImmediately().load("sqlncli11");
49
+ } catch (UnsatisfiedLinkError e) {
50
+ throw new RuntimeException("sqlncli11.dll not found.", e);
51
+ }
52
+ }
53
+ }
54
+
55
+ charset = Charset.forName("MS932");
56
+ wideCharset = Charset.forName("UTF-16LE");
57
+ }
58
+
59
+ public void open(String server, int port, Optional<String> instance,
60
+ String database, Optional<String> user, Optional<String> password,
61
+ String table)
62
+ throws SQLException
63
+ {
64
+ // environment handle
65
+ Pointer envHandlePointer = createPointerPointer();
66
+ checkSQLResult("SQLAllocHandle(SQL_HANDLE_ENV)", odbc.SQLAllocHandle(
67
+ ODBC.SQL_HANDLE_ENV,
68
+ null,
69
+ envHandlePointer));
70
+ envHandle = envHandlePointer.getPointer(0);
71
+
72
+ // set ODBC version
73
+ checkSQLResult("SQLSetEnvAttr(SQL_ATTR_ODBC_VERSION)", odbc.SQLSetEnvAttr(
74
+ envHandle,
75
+ ODBC.SQL_ATTR_ODBC_VERSION,
76
+ Pointer.wrap(Runtime.getSystemRuntime(), ODBC.SQL_OV_ODBC3),
77
+ ODBC.SQL_IS_INTEGER));
78
+
79
+ // ODBC handle
80
+ Pointer odbcHandlePointer = createPointerPointer();
81
+ checkSQLResult("SQLAllocHandle(SQL_HANDLE_DBC)", odbc.SQLAllocHandle(
82
+ ODBC.SQL_HANDLE_DBC,
83
+ envHandle,
84
+ odbcHandlePointer));
85
+ odbcHandle = odbcHandlePointer.getPointer(0);
86
+
87
+ // set BULK COPY mode
88
+ checkSQLResult("SQLSetConnectAttr(SQL_COPT_SS_BCP)", odbc.SQLSetConnectAttrW(
89
+ odbcHandle,
90
+ ODBC.SQL_COPT_SS_BCP,
91
+ Pointer.wrap(Runtime.getSystemRuntime(), ODBC.SQL_BCP_ON),
92
+ ODBC.SQL_IS_INTEGER));
93
+
94
+ StringBuilder connectionString = new StringBuilder();
95
+ connectionString.append("Driver={SQL Server Native Client 11.0};");
96
+ if (instance.isPresent()) {
97
+ connectionString.append(String.format("Server=%s,%d\\%s;", server, port, instance.get()));
98
+ } else {
99
+ connectionString.append(String.format("Server=%s,%d;", server, port));
100
+ }
101
+ connectionString.append(String.format("Database=%s;", database));
102
+ if (user.isPresent()) {
103
+ connectionString.append(String.format("UID=%s;", user.get()));
104
+ }
105
+ if (password.isPresent()) {
106
+ logger.info("connection string = " + connectionString + "PWD=********;");
107
+ connectionString.append(String.format("PWD=%s;", password.get()));
108
+ } else {
109
+ logger.info("connection string = " + connectionString);
110
+ }
111
+
112
+ checkSQLResult("SQLDriverConnect", odbc.SQLDriverConnectW(
113
+ odbcHandle,
114
+ null,
115
+ toWideChars(connectionString.toString()),
116
+ ODBC.SQL_NTS,
117
+ null,
118
+ ODBC.SQL_NTS,
119
+ null,
120
+ ODBC.SQL_DRIVER_NOPROMPT));
121
+
122
+ StringBuilder fullTableName = new StringBuilder();
123
+ fullTableName.append("[");
124
+ fullTableName.append(database);
125
+ fullTableName.append("].");
126
+ fullTableName.append(".[");
127
+ fullTableName.append(table);
128
+ fullTableName.append("]");
129
+ checkBCPResult("bcp_init", client.bcp_initW(
130
+ odbcHandle,
131
+ toWideChars(fullTableName.toString()),
132
+ null,
133
+ null,
134
+ NativeClient.DB_IN));
135
+ }
136
+
137
+ public int bindNull(int columnIndex) throws SQLException
138
+ {
139
+ Pointer pointer = prepareBuffer(columnIndex, 0);
140
+ checkBCPResult("bcp_bind", client.bcp_bind(
141
+ odbcHandle,
142
+ pointer,
143
+ 0,
144
+ NativeClient.SQL_NULL_DATA,
145
+ null,
146
+ 0,
147
+ NativeClient.SQLCHARACTER,
148
+ columnIndex));
149
+ return (int)pointer.size();
150
+ }
151
+
152
+ public int bindValue(int columnIndex, String value) throws SQLException
153
+ {
154
+ ByteBuffer bytes = charset.encode(value);
155
+ Pointer pointer = prepareBuffer(columnIndex, bytes.remaining());
156
+ pointer.put(0, bytes.array(), 0, bytes.remaining());
157
+
158
+ checkBCPResult("bcp_bind", client.bcp_bind(
159
+ odbcHandle,
160
+ pointer,
161
+ 0,
162
+ (int)pointer.size(),
163
+ null,
164
+ 0,
165
+ NativeClient.SQLCHARACTER,
166
+ columnIndex));
167
+ return (int)pointer.size();
168
+ }
169
+
170
+ public int bindValue(int columnIndex, boolean value) throws SQLException
171
+ {
172
+ Pointer pointer = prepareBuffer(columnIndex, 1);
173
+ pointer.putByte(0, value ? (byte)1 : (byte)0);
174
+
175
+ checkBCPResult("bcp_bind", client.bcp_bind(
176
+ odbcHandle,
177
+ pointer,
178
+ 0,
179
+ (int)pointer.size(),
180
+ null,
181
+ 0,
182
+ NativeClient.SQLBIT,
183
+ columnIndex));
184
+ return (int)pointer.size();
185
+ }
186
+
187
+ public int bindValue(int columnIndex, byte value) throws SQLException
188
+ {
189
+ Pointer pointer = prepareBuffer(columnIndex, 1);
190
+ pointer.putByte(0, value);
191
+
192
+ checkBCPResult("bcp_bind", client.bcp_bind(
193
+ odbcHandle,
194
+ pointer,
195
+ 0,
196
+ (int)pointer.size(),
197
+ null,
198
+ 0,
199
+ NativeClient.SQLINT1,
200
+ columnIndex));
201
+ return (int)pointer.size();
202
+ }
203
+
204
+ public int bindValue(int columnIndex, short value) throws SQLException
205
+ {
206
+ Pointer pointer = prepareBuffer(columnIndex, 2);
207
+ pointer.putShort(0, value);
208
+
209
+ checkBCPResult("bcp_bind", client.bcp_bind(
210
+ odbcHandle,
211
+ pointer,
212
+ 0,
213
+ (int)pointer.size(),
214
+ null,
215
+ 0,
216
+ NativeClient.SQLINT2,
217
+ columnIndex));
218
+ return (int)pointer.size();
219
+ }
220
+
221
+ public int bindValue(int columnIndex, int value) throws SQLException
222
+ {
223
+ Pointer pointer = prepareBuffer(columnIndex, 4);
224
+ pointer.putInt(0, value);
225
+
226
+ checkBCPResult("bcp_bind", client.bcp_bind(
227
+ odbcHandle,
228
+ pointer,
229
+ 0,
230
+ (int)pointer.size(),
231
+ null,
232
+ 0,
233
+ NativeClient.SQLINT4,
234
+ columnIndex));
235
+ return (int)pointer.size();
236
+ }
237
+
238
+ public int bindValue(int columnIndex, long value) throws SQLException
239
+ {
240
+ Pointer pointer = prepareBuffer(columnIndex, 8);
241
+ pointer.putLongLong(0, value);
242
+
243
+ checkBCPResult("bcp_bind", client.bcp_bind(
244
+ odbcHandle,
245
+ pointer,
246
+ 0,
247
+ (int)pointer.size(),
248
+ null,
249
+ 0,
250
+ NativeClient.SQLINT8,
251
+ columnIndex));
252
+ return (int)pointer.size();
253
+ }
254
+
255
+ public int bindValue(int columnIndex, float value) throws SQLException
256
+ {
257
+ Pointer pointer = prepareBuffer(columnIndex, 4);
258
+ pointer.putFloat(0, value);
259
+
260
+ checkBCPResult("bcp_bind", client.bcp_bind(
261
+ odbcHandle,
262
+ pointer,
263
+ 0,
264
+ (int)pointer.size(),
265
+ null,
266
+ 0,
267
+ NativeClient.SQLFLT4,
268
+ columnIndex));
269
+ return (int)pointer.size();
270
+ }
271
+
272
+ public int bindValue(int columnIndex, double value) throws SQLException
273
+ {
274
+ Pointer pointer = prepareBuffer(columnIndex, 8);
275
+ pointer.putDouble(0, value);
276
+
277
+ checkBCPResult("bcp_bind", client.bcp_bind(
278
+ odbcHandle,
279
+ pointer,
280
+ 0,
281
+ (int)pointer.size(),
282
+ null,
283
+ 0,
284
+ NativeClient.SQLFLT8,
285
+ columnIndex));
286
+ return (int)pointer.size();
287
+ }
288
+
289
+ private Pointer prepareBuffer(int columnIndex, int size)
290
+ {
291
+ Pointer pointer = boundPointers.get(columnIndex);
292
+ if (pointer == null || pointer.size() < size) {
293
+ Runtime runtime = Runtime.getSystemRuntime();
294
+ pointer = Pointer.wrap(runtime, ByteBuffer.allocateDirect(size).order(runtime.byteOrder()));
295
+ boundPointers.put(columnIndex, pointer);
296
+ }
297
+ return pointer;
298
+ }
299
+
300
+ public void sendRow() throws SQLException
301
+ {
302
+ checkBCPResult("bcp_sendrow", client.bcp_sendrow(odbcHandle));
303
+ }
304
+
305
+ public void commit(boolean done) throws SQLException
306
+ {
307
+ String operation;
308
+ int result;
309
+ if (done) {
310
+ operation = "bcp_done";
311
+ result = client.bcp_done(odbcHandle);
312
+ } else {
313
+ operation = "bcp_batch";
314
+ result = client.bcp_batch(odbcHandle);
315
+ }
316
+ if (result < 0) {
317
+ throwException(operation, NativeClient.FAIL);
318
+ } else {
319
+ if (result > 0) {
320
+ logger.info(String.format("SQL Server Native Client : %,d rows have bean loaded.", result));
321
+ }
322
+ }
323
+
324
+ }
325
+
326
+ public void close()
327
+ {
328
+ if (odbcHandle != null) {
329
+ odbc.SQLFreeHandle(ODBC.SQL_HANDLE_DBC, odbcHandle);
330
+ odbcHandle = null;
331
+ }
332
+ if (envHandle != null) {
333
+ odbc.SQLFreeHandle(ODBC.SQL_HANDLE_ENV, envHandle);
334
+ envHandle = null;
335
+ }
336
+ }
337
+
338
+ private Pointer createPointerPointer()
339
+ {
340
+ return new ArrayMemoryIO(Runtime.getSystemRuntime(), com.kenai.jffi.Type.POINTER.size());
341
+ }
342
+
343
+ private String toString(Pointer wcharPointer, int length)
344
+ {
345
+ byte[] bytes = new byte[length * 2];
346
+ wcharPointer.get(0, bytes, 0, length * 2);
347
+ CharBuffer chars = wideCharset.decode(ByteBuffer.wrap(bytes));
348
+ return chars.toString();
349
+ }
350
+
351
+ private Pointer toWideChars(String s)
352
+ {
353
+ ByteBuffer bytes = wideCharset.encode(s);
354
+ Pointer pointer = new ArrayMemoryIO(Runtime.getSystemRuntime(), bytes.remaining() + 2);
355
+ pointer.put(0, bytes.array(), 0, bytes.remaining());
356
+ pointer.putShort(bytes.remaining(), (short)0);
357
+ return pointer;
358
+ }
359
+
360
+ private void checkSQLResult(String operation, short result) throws SQLException
361
+ {
362
+ switch (result) {
363
+ case ODBC.SQL_SUCCESS:
364
+ break;
365
+
366
+ case ODBC.SQL_SUCCESS_WITH_INFO:
367
+ StringBuilder sqlState = new StringBuilder();
368
+ StringBuilder sqlMessage = new StringBuilder();
369
+ if (getErrorMessage(sqlState, sqlMessage)) {
370
+ logger.info(String.format("SQL Server Native Client : %s : %s", operation, sqlMessage));
371
+ }
372
+ break;
373
+
374
+ default:
375
+ throwException(operation, result);
376
+ }
377
+ }
378
+
379
+ private void checkBCPResult(String operation, short result) throws SQLException
380
+ {
381
+ switch (result) {
382
+ case NativeClient.SUCCEED:
383
+ break;
384
+
385
+ default:
386
+ throwException(operation, result);
387
+ }
388
+ }
389
+
390
+ private void throwException(String operation, short result) throws SQLException
391
+ {
392
+ String message = String.format("SQL Server Native Client : %s failed : %d.", operation, result);
393
+
394
+ if (odbcHandle != null) {
395
+ StringBuilder sqlState = new StringBuilder();
396
+ StringBuilder sqlMessage = new StringBuilder();
397
+ if (getErrorMessage(sqlState, sqlMessage)) {
398
+ message = String.format("SQL Server Native Client : %s failed (sql state = %s) : %s", operation, sqlState, sqlMessage);
399
+ }
400
+ }
401
+
402
+ logger.error(message);
403
+ throw new SQLException(message);
404
+ }
405
+
406
+ private boolean getErrorMessage(StringBuilder sqlState, StringBuilder sqlMessage)
407
+ {
408
+ final int sqlStateLength = 5;
409
+ // (5 (SQL state length) + 1 (terminator length)) * 2 (wchar size)
410
+ Pointer sqlStatePointer = new ArrayMemoryIO(Runtime.getSystemRuntime(), (sqlStateLength + 1) * 2);
411
+ Pointer sqlMessagePointer = new ArrayMemoryIO(Runtime.getSystemRuntime(), 512);
412
+ Pointer lengthPointer = new ArrayMemoryIO(Runtime.getSystemRuntime(), 4);
413
+
414
+ for (short record = 1;; record++) {
415
+ short result = odbc.SQLGetDiagRecW(
416
+ ODBC.SQL_HANDLE_DBC,
417
+ odbcHandle,
418
+ record,
419
+ sqlStatePointer,
420
+ null,
421
+ sqlMessagePointer,
422
+ (short)(sqlMessagePointer.size() / 2),
423
+ lengthPointer);
424
+
425
+ if (result == ODBC.SQL_SUCCESS) {
426
+ if (record > 1) {
427
+ sqlState.append(",");
428
+ }
429
+ sqlState.append(toString(sqlStatePointer, sqlStateLength));
430
+ sqlMessage.append(toString(sqlMessagePointer, lengthPointer.getInt(0)));
431
+ } else {
432
+ if (record == 1) {
433
+ return false;
434
+ }
435
+ break;
436
+ }
437
+ }
438
+
439
+ return true;
440
+ }
441
+
442
+ }
@@ -0,0 +1,47 @@
1
+ package org.embulk.output.sqlserver.nativeclient;
2
+
3
+ import jnr.ffi.Pointer;
4
+
5
+
6
+ /**
7
+ * sqlncli11.dll also has SQLXXX methods, but we should call those methods in odbc32.dll .
8
+ */
9
+ public interface ODBC
10
+ {
11
+ static short SQL_SUCCESS = 0;
12
+ static short SQL_SUCCESS_WITH_INFO = 1;
13
+
14
+ static short SQL_HANDLE_ENV = 1;
15
+ static short SQL_HANDLE_DBC = 2;
16
+
17
+ static short SQL_ATTR_ODBC_VERSION = 200;
18
+
19
+ static short SQL_COPT_SS_BASE = 1200;
20
+ static short SQL_COPT_SS_BCP = SQL_COPT_SS_BASE + 19;
21
+
22
+ static short SQL_DRIVER_NOPROMPT = 0;
23
+
24
+ static long SQL_OV_ODBC3 = 3;
25
+ static long SQL_BCP_ON = 1;
26
+
27
+ static int SQL_IS_INTEGER = -6;
28
+ static short SQL_NTS = -3;
29
+
30
+
31
+ short SQLAllocHandle(short handleType, Pointer inputHandle, Pointer outputHandle);
32
+
33
+ short SQLSetEnvAttr(Pointer environmentHandle, short attribute, Pointer value, int stringLength);
34
+
35
+ short SQLSetConnectAttrW(Pointer hdbc, int fAttribute, Pointer rgbValue, int cbValue);
36
+
37
+ short SQLDriverConnectW(Pointer hdbc, Pointer hwnd,
38
+ Pointer szConnStrIn, short cchConnStrIn,
39
+ Pointer szConnStrOut, short cchConnStrOutMax, Pointer pcchConnStrOut,
40
+ short fDriverCompletion);
41
+
42
+ short SQLFreeHandle(short handleType, Pointer handle);
43
+
44
+ short SQLGetDiagRecW(short handleType, Pointer handle, short recNumber,
45
+ Pointer sqlState, Pointer nativeErrorPtr,
46
+ Pointer messageText, short bufferLength, Pointer textLengthPtr);
47
+ }