embulk-output-sqlserver 0.5.0 → 0.5.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (23) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -3
  3. data/classpath/{embulk-output-jdbc-0.5.0.jar → embulk-output-jdbc-0.5.1.jar} +0 -0
  4. data/classpath/embulk-output-sqlserver-0.5.1.jar +0 -0
  5. data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java +22 -4
  6. data/src/main/java/org/embulk/output/sqlserver/InsertMethod.java +32 -0
  7. data/src/main/java/org/embulk/output/sqlserver/NativeBatchInsert.java +247 -0
  8. data/src/main/java/org/embulk/output/sqlserver/SmallDateTimeFormat.java +31 -0
  9. data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClient.java +33 -0
  10. data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClientWrapper.java +442 -0
  11. data/src/main/java/org/embulk/output/sqlserver/nativeclient/ODBC.java +47 -0
  12. data/src/test/java/org/embulk/output/sqlserver/SQLServerOutputPluginTest.java +174 -10
  13. data/src/test/resources/sqlserver/data/test2/test2.csv +2 -0
  14. data/src/test/resources/sqlserver/data/test3/test3.csv +2 -0
  15. data/src/test/resources/sqlserver/data/test4/test4.csv +2 -0
  16. data/src/test/resources/sqlserver/data/test5/test5.csv +2 -0
  17. data/src/test/resources/sqlserver/yml/test-native-date.yml +26 -0
  18. data/src/test/resources/sqlserver/yml/test-native-decimal.yml +25 -0
  19. data/src/test/resources/sqlserver/yml/test-native-integer.yml +24 -0
  20. data/src/test/resources/sqlserver/yml/test-native-string.yml +25 -0
  21. data/src/test/resources/sqlserver/yml/test-native.yml +44 -0
  22. metadata +19 -4
  23. data/classpath/embulk-output-sqlserver-0.5.0.jar +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 76d8fb06a79d4382fab1f378b165ab10518fe9ef
4
- data.tar.gz: cc766e5eeff96a7a1e74e00893a01beb58a13e83
3
+ metadata.gz: 107530637391bd44da5f7291004ccef1e1ba0a3b
4
+ data.tar.gz: a89e2a72c151e984261519e31d879d5449db9060
5
5
  SHA512:
6
- metadata.gz: e39ecf1f59e3b3c555733821925e0a098be114b1d15844c6bcb1f8349de93168cb46bbfc999f427f5231f061d89bfae509e875d9d480e366224e639385dccd54
7
- data.tar.gz: 45419b194e670602d291ff1e1ee5ecedb0c7a6d9b236340e056ce82a37b1cdc7838dd01cc03ebafa41470f10b5cef673ea25f85a902b4a625ffaab7501a00968
6
+ metadata.gz: b6a2ce12e01cbe02743d0ca42de24bf1105ab3ff40ab333cf56f198877ca967577792af1f8b3d6ee0fdaae02894788b7aef66a6aabed86db0c99883459c7b6f5
7
+ data.tar.gz: 96aeaf1f4c005375a4ce991f6dc9eef502bc896aa3ddb69ed2c492c34809e1b5468726a71adbc7d793a8895988e442db87ff9d3e88fefb8eb9ad6d0a6d7ddfd8
data/README.md CHANGED
@@ -5,8 +5,8 @@ SQL Server output plugins for Embulk loads records to SQL Server.
5
5
  ## Overview
6
6
 
7
7
  * **Plugin type**: output
8
- * **Load all or nothing**: depnds on the mode. see bellow.
9
- * **Resume supported**: depnds on the mode. see bellow.
8
+ * **Load all or nothing**: depnds on the mode. see below.
9
+ * **Resume supported**: depnds on the mode. see below.
10
10
 
11
11
  ## Configuration
12
12
 
@@ -25,7 +25,8 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
25
25
  - **url**: URL of the JDBC connection (string, optional)
26
26
  - **table**: destination table name (string, required)
27
27
  - **options**: extra connection properties (hash, default: {})
28
- - **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See bellow. (string, required)
28
+ - **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See below. (string, required)
29
+ - **insert_method**: see below
29
30
  - **batch_size**: size of a single batch insert (integer, default: 16777216)
30
31
  - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
31
32
  - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -53,6 +54,15 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
53
54
  * Transactional: No. If fails, the target table could be dropped (because SQL Server can't rollback DDL).
54
55
  * Resumable: No.
55
56
 
57
+ ### Insert methods
58
+
59
+ insert_method supports three options.
60
+
61
+ "normal" means normal insert (default). It requires SQL Server JDBC driver.
62
+
63
+ "native" means bulk insert using native client. It is faster than "normal".
64
+ It requires both SQL Server JDBC driver and SQL Server Native Client (11.0).
65
+
56
66
  ### Example
57
67
 
58
68
  ```yaml
@@ -81,6 +91,7 @@ out:
81
91
  database: my_database
82
92
  table: my_table
83
93
  mode: insert_direct
94
+ insert_method: native
84
95
  column_options:
85
96
  my_col_1: {type: 'TEXT'}
86
97
  my_col_3: {type: 'INT NOT NULL'}
@@ -11,6 +11,8 @@ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
11
11
  import org.embulk.output.jdbc.BatchInsert;
12
12
  import org.embulk.output.jdbc.StandardBatchInsert;
13
13
  import org.embulk.output.jdbc.setter.ColumnSetterFactory;
14
+ import org.embulk.output.sqlserver.InsertMethod;
15
+ import org.embulk.output.sqlserver.NativeBatchInsert;
14
16
  import org.embulk.output.sqlserver.SQLServerOutputConnector;
15
17
  import org.embulk.output.sqlserver.setter.SQLServerColumnSetterFactory;
16
18
  import org.joda.time.DateTimeZone;
@@ -60,6 +62,9 @@ public class SQLServerOutputPlugin
60
62
  @ConfigDefault("\"\"")
61
63
  public Optional<String> getPassword();
62
64
 
65
+ @Config("insert_method")
66
+ @ConfigDefault("\"normal\"")
67
+ public InsertMethod getInsertMethod();
63
68
  }
64
69
 
65
70
  @Override
@@ -88,6 +93,10 @@ public class SQLServerOutputPlugin
88
93
 
89
94
  String url;
90
95
  if (sqlServerTask.getUrl().isPresent()) {
96
+ if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
97
+ throw new IllegalArgumentException("Cannot set 'url' when 'insert_method' is 'native'.");
98
+ }
99
+
91
100
  if (sqlServerTask.getHost().isPresent()
92
101
  || sqlServerTask.getInstance().isPresent()
93
102
  || sqlServerTask.getDatabase().isPresent()
@@ -104,8 +113,8 @@ public class SQLServerOutputPlugin
104
113
  }
105
114
  StringBuilder urlBuilder = new StringBuilder();
106
115
  if (sqlServerTask.getInstance().isPresent()) {
107
- urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s:%d",
108
- sqlServerTask.getHost().get(), sqlServerTask.getInstance().get(), sqlServerTask.getPort()));
116
+ urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s",
117
+ sqlServerTask.getHost().get(), sqlServerTask.getInstance().get()));
109
118
  } else {
110
119
  urlBuilder.append(String.format("jdbc:sqlserver://%s:%d",
111
120
  sqlServerTask.getHost().get(), sqlServerTask.getPort()));
@@ -130,9 +139,13 @@ public class SQLServerOutputPlugin
130
139
  Properties props = new Properties();
131
140
  props.putAll(sqlServerTask.getOptions());
132
141
 
133
- props.setProperty("user", sqlServerTask.getUser().get());
142
+ if (sqlServerTask.getUser().isPresent()) {
143
+ props.setProperty("user", sqlServerTask.getUser().get());
144
+ }
134
145
  logger.info("Connecting to {} options {}", url, props);
135
- props.setProperty("password", sqlServerTask.getPassword().get());
146
+ if (sqlServerTask.getPassword().isPresent()) {
147
+ props.setProperty("password", sqlServerTask.getPassword().get());
148
+ }
136
149
 
137
150
  return new SQLServerOutputConnector(url, props, null);
138
151
  }
@@ -140,6 +153,11 @@ public class SQLServerOutputPlugin
140
153
  @Override
141
154
  protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
142
155
  {
156
+ SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
157
+ if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
158
+ return new NativeBatchInsert(sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
159
+ sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword());
160
+ }
143
161
  return new StandardBatchInsert(getConnector(task, true), mergeKeys);
144
162
  }
145
163
 
@@ -0,0 +1,32 @@
1
+ package org.embulk.output.sqlserver;
2
+
3
+ import java.util.Locale;
4
+
5
+ import org.embulk.config.ConfigException;
6
+
7
+ import com.fasterxml.jackson.annotation.JsonCreator;
8
+ import com.fasterxml.jackson.annotation.JsonValue;
9
+
10
+ public enum InsertMethod
11
+ {
12
+ NORMAL,
13
+ NATIVE;
14
+
15
+ @JsonValue
16
+ @Override
17
+ public String toString()
18
+ {
19
+ return name().toLowerCase(Locale.ENGLISH);
20
+ }
21
+
22
+ @JsonCreator
23
+ public static InsertMethod fromString(String value)
24
+ {
25
+ for (InsertMethod insertMethod : InsertMethod.values()) {
26
+ if (insertMethod.toString().equals(value)) {
27
+ return insertMethod;
28
+ }
29
+ }
30
+ throw new ConfigException(String.format("Unknown insert_method '%s'.", value));
31
+ }
32
+ }
@@ -0,0 +1,247 @@
1
+ package org.embulk.output.sqlserver;
2
+
3
+ import java.io.IOException;
4
+ import java.math.BigDecimal;
5
+ import java.sql.SQLException;
6
+ import java.sql.Types;
7
+ import java.text.DateFormat;
8
+ import java.text.SimpleDateFormat;
9
+ import java.util.Calendar;
10
+
11
+ import org.embulk.output.jdbc.BatchInsert;
12
+ import org.embulk.output.jdbc.JdbcColumn;
13
+ import org.embulk.output.jdbc.JdbcSchema;
14
+ import org.embulk.output.jdbc.StandardBatchInsert;
15
+ import org.embulk.output.jdbc.TimestampFormat;
16
+ import org.embulk.output.sqlserver.nativeclient.NativeClientWrapper;
17
+ import org.embulk.spi.Exec;
18
+ import org.embulk.spi.time.Timestamp;
19
+ import org.slf4j.Logger;
20
+
21
+ import com.google.common.base.Optional;
22
+
23
+ public class NativeBatchInsert implements BatchInsert
24
+ {
25
+ private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
26
+
27
+ private NativeClientWrapper client = new NativeClientWrapper();
28
+
29
+ private final String server;
30
+ private final int port;
31
+ private final Optional<String> instance;
32
+ private final String database;
33
+ private final Optional<String> user;
34
+ private final Optional<String> password;
35
+
36
+ private int batchWeight;
37
+ private int batchRows;
38
+ private long totalRows;
39
+
40
+ private int columnCount;
41
+ private int lastColumnIndex;
42
+
43
+ private DateFormat[] formats;
44
+
45
+
46
+ public NativeBatchInsert(String server, int port, Optional<String> instance,
47
+ String database, Optional<String> user, Optional<String> password)
48
+ {
49
+ this.server = server;
50
+ this.port = port;
51
+ this.instance = instance;
52
+ this.database = database;
53
+ this.user = user;
54
+ this.password = password;
55
+
56
+ lastColumnIndex = 0;
57
+ }
58
+
59
+
60
+ @Override
61
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
62
+ {
63
+ columnCount = insertSchema.getCount();
64
+ client.open(server, port, instance, database, user, password, loadTable);
65
+
66
+ formats = new DateFormat[insertSchema.getCount()];
67
+ for (int i = 0; i < insertSchema.getCount(); i++) {
68
+ JdbcColumn column = insertSchema.getColumn(i);
69
+ switch (column.getSqlType()) {
70
+ case Types.DATE:
71
+ formats[i] = new SimpleDateFormat("yyyy-MM-dd");
72
+ break;
73
+
74
+ case Types.TIME:
75
+ formats[i] = new TimestampFormat("HH:mm:ss", column.getScaleTypeParameter());
76
+ break;
77
+
78
+ case Types.TIMESTAMP:
79
+ if (column.getSimpleTypeName().equals("SMALLDATETIME")) {
80
+ formats[i] = new SmallDateTimeFormat("yyyy-MM-dd HH:mm:ss");
81
+ } else {
82
+ formats[i] = new TimestampFormat("yyyy-MM-dd HH:mm:ss", column.getScaleTypeParameter());
83
+ }
84
+ break;
85
+
86
+ default:
87
+ break;
88
+ }
89
+ }
90
+ }
91
+
92
+ @Override
93
+ public int getBatchWeight()
94
+ {
95
+ return batchWeight;
96
+ }
97
+
98
+ @Override
99
+ public void add() throws IOException, SQLException
100
+ {
101
+ client.sendRow();
102
+
103
+ batchRows++;
104
+ batchWeight += 32; // add weight as overhead of each rows
105
+ }
106
+
107
+ private int nextColumnIndex()
108
+ {
109
+ int nextColumnIndex = lastColumnIndex + 1;
110
+ if (nextColumnIndex == columnCount) {
111
+ lastColumnIndex = 0;
112
+ } else {
113
+ lastColumnIndex++;
114
+ }
115
+ return nextColumnIndex;
116
+ }
117
+
118
+ @Override
119
+ public void setNull(int sqlType) throws IOException, SQLException
120
+ {
121
+ batchWeight += client.bindNull(nextColumnIndex());
122
+ }
123
+
124
+ @Override
125
+ public void setBoolean(boolean v) throws IOException, SQLException
126
+ {
127
+ batchWeight += client.bindValue(nextColumnIndex(), v);
128
+ }
129
+
130
+ @Override
131
+ public void setByte(byte v) throws IOException, SQLException
132
+ {
133
+ batchWeight += client.bindValue(nextColumnIndex(), v);
134
+ }
135
+
136
+ @Override
137
+ public void setShort(short v) throws IOException, SQLException
138
+ {
139
+ batchWeight += client.bindValue(nextColumnIndex(), v);
140
+ }
141
+
142
+ @Override
143
+ public void setInt(int v) throws IOException, SQLException
144
+ {
145
+ batchWeight += client.bindValue(nextColumnIndex(), v);
146
+ }
147
+
148
+ @Override
149
+ public void setLong(long v) throws IOException, SQLException
150
+ {
151
+ batchWeight += client.bindValue(nextColumnIndex(), v);
152
+ }
153
+
154
+ @Override
155
+ public void setFloat(float v) throws IOException, SQLException
156
+ {
157
+ batchWeight += client.bindValue(nextColumnIndex(), v);
158
+ }
159
+
160
+ @Override
161
+ public void setDouble(double v) throws IOException, SQLException
162
+ {
163
+ batchWeight += client.bindValue(nextColumnIndex(), v);
164
+ }
165
+
166
+ @Override
167
+ public void setBigDecimal(BigDecimal v) throws IOException, SQLException
168
+ {
169
+ batchWeight += client.bindValue(nextColumnIndex(), v.toPlainString());
170
+ }
171
+
172
+ @Override
173
+ public void setString(String v) throws IOException, SQLException
174
+ {
175
+ batchWeight += client.bindValue(nextColumnIndex(), v);
176
+ }
177
+
178
+ @Override
179
+ public void setNString(String v) throws IOException, SQLException
180
+ {
181
+ batchWeight += client.bindValue(nextColumnIndex(), v);
182
+ }
183
+
184
+ @Override
185
+ public void setBytes(byte[] v) throws IOException, SQLException
186
+ {
187
+ throw new SQLException("Unsupported");
188
+ }
189
+
190
+ @Override
191
+ public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
192
+ {
193
+ setSqlTimestamp(v, cal);
194
+ }
195
+
196
+ @Override
197
+ public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
198
+ {
199
+ setSqlTimestamp(v, cal);
200
+ }
201
+
202
+ @Override
203
+ public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
204
+ {
205
+ int columnIndex = nextColumnIndex();
206
+ DateFormat format = formats[columnIndex - 1];
207
+ format.setCalendar(cal);
208
+
209
+ java.sql.Timestamp timestamp = new java.sql.Timestamp(v.toEpochMilli());
210
+ timestamp.setNanos(v.getNano());
211
+
212
+ batchWeight += client.bindValue(columnIndex, format.format(timestamp));
213
+ }
214
+
215
+ @Override
216
+ public void flush() throws IOException, SQLException
217
+ {
218
+ logger.info(String.format("Loading %,d rows", batchRows));
219
+ long startTime = System.currentTimeMillis();
220
+
221
+ client.commit(false);
222
+
223
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
224
+ totalRows += batchRows;
225
+
226
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
227
+
228
+ batchRows = 0;
229
+ batchWeight = 0;
230
+ }
231
+
232
+ @Override
233
+ public void finish() throws IOException, SQLException
234
+ {
235
+ if (getBatchWeight() != 0) {
236
+ flush();
237
+ }
238
+ client.commit(true);
239
+ }
240
+
241
+ @Override
242
+ public void close() throws IOException, SQLException
243
+ {
244
+ client.close();
245
+ }
246
+
247
+ }
@@ -0,0 +1,31 @@
1
+ package org.embulk.output.sqlserver;
2
+
3
+ import java.text.FieldPosition;
4
+ import java.text.SimpleDateFormat;
5
+ import java.util.Date;
6
+
7
+
8
+ public class SmallDateTimeFormat extends SimpleDateFormat
9
+ {
10
+ public SmallDateTimeFormat(String pattern)
11
+ {
12
+ super(pattern);
13
+ }
14
+
15
+ @Override
16
+ public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos)
17
+ {
18
+ long time = date.getTime();
19
+
20
+ // round seconds
21
+ long underMinutes = time % 60000;
22
+ if (underMinutes < 30000) {
23
+ time -= underMinutes;
24
+ } else {
25
+ time += 60000 - underMinutes;
26
+ }
27
+
28
+ return super.format(new Date(time), toAppendTo, pos);
29
+ }
30
+
31
+ }
@@ -0,0 +1,33 @@
1
+ package org.embulk.output.sqlserver.nativeclient;
2
+
3
+ import jnr.ffi.Pointer;
4
+
5
+ public interface NativeClient
6
+ {
7
+ static int SQL_NULL_DATA = -1;
8
+ static int SQLCHARACTER = 0x2F;
9
+ static int SQLINT1 = 0x30;
10
+ static int SQLBIT = 0x32;
11
+ static int SQLINT2 = 0x34;
12
+ static int SQLINT4 = 0x38;
13
+ static int SQLFLT8 = 0x3E;
14
+ static int SQLFLT4 = 0x3B;
15
+ static int SQLINT8 = 0x7F;
16
+
17
+ static short FAIL = 0;
18
+ static short SUCCEED = 1;
19
+ static int DB_IN = 1;
20
+
21
+ short bcp_initW(Pointer hdbc, Pointer szTable, Pointer szDataFile, Pointer szErrorFile, int eDirection);
22
+
23
+ short bcp_bind(Pointer hdbc,
24
+ Pointer pData, int cbIndicator, int cbData,
25
+ Pointer pTerm, int cbTerm,
26
+ int eDataType, int idxServerCol);
27
+
28
+ short bcp_sendrow(Pointer hdbc);
29
+
30
+ int bcp_batch(Pointer hdbc);
31
+
32
+ int bcp_done(Pointer hdbc);
33
+ }