embulk-output-jdbc 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +2 -2
- data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
- data/lib/embulk/output/jdbc.rb +3 -3
- data/src/main/java/org/embulk/output/JdbcOutputPlugin.java +137 -138
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +972 -973
- data/src/main/java/org/embulk/output/jdbc/BatchInsert.java +52 -53
- data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java +116 -116
- data/src/main/java/org/embulk/output/jdbc/JdbcColumnOption.java +34 -34
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +492 -492
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java +8 -8
- data/src/main/java/org/embulk/output/jdbc/JdbcSchema.java +60 -60
- data/src/main/java/org/embulk/output/jdbc/JdbcUtils.java +153 -155
- data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +200 -200
- data/src/main/java/org/embulk/output/jdbc/ToString.java +54 -54
- data/src/main/java/org/embulk/output/jdbc/ToStringMap.java +34 -35
- data/src/main/java/org/embulk/output/jdbc/setter/BigDecimalColumnSetter.java +68 -70
- data/src/main/java/org/embulk/output/jdbc/setter/BooleanColumnSetter.java +53 -54
- data/src/main/java/org/embulk/output/jdbc/setter/ByteColumnSetter.java +75 -76
- data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java +44 -45
- data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterFactory.java +196 -196
- data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterVisitor.java +95 -96
- data/src/main/java/org/embulk/output/jdbc/setter/DefaultValueSetter.java +48 -48
- data/src/main/java/org/embulk/output/jdbc/setter/DoubleColumnSetter.java +60 -61
- data/src/main/java/org/embulk/output/jdbc/setter/FloatColumnSetter.java +60 -61
- data/src/main/java/org/embulk/output/jdbc/setter/IntColumnSetter.java +75 -76
- data/src/main/java/org/embulk/output/jdbc/setter/LongColumnSetter.java +71 -72
- data/src/main/java/org/embulk/output/jdbc/setter/NStringColumnSetter.java +58 -59
- data/src/main/java/org/embulk/output/jdbc/setter/NullColumnSetter.java +53 -53
- data/src/main/java/org/embulk/output/jdbc/setter/NullDefaultValueSetter.java +105 -105
- data/src/main/java/org/embulk/output/jdbc/setter/PassThroughColumnSetter.java +58 -59
- data/src/main/java/org/embulk/output/jdbc/setter/ShortColumnSetter.java +75 -76
- data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java +43 -43
- data/src/main/java/org/embulk/output/jdbc/setter/SqlDateColumnSetter.java +58 -59
- data/src/main/java/org/embulk/output/jdbc/setter/SqlTimeColumnSetter.java +58 -59
- data/src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java +58 -58
- data/src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java +58 -59
- data/src/test/java/org/embulk/output/TestJdbcOutputPlugin.java +5 -5
- metadata +3 -3
- data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
@@ -1,8 +1,8 @@
|
|
1
|
-
package org.embulk.output.jdbc;
|
2
|
-
|
3
|
-
import java.sql.SQLException;
|
4
|
-
|
5
|
-
public interface JdbcOutputConnector
|
6
|
-
{
|
7
|
-
public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
|
8
|
-
}
|
1
|
+
package org.embulk.output.jdbc;
|
2
|
+
|
3
|
+
import java.sql.SQLException;
|
4
|
+
|
5
|
+
public interface JdbcOutputConnector
|
6
|
+
{
|
7
|
+
public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
|
8
|
+
}
|
@@ -1,60 +1,60 @@
|
|
1
|
-
package org.embulk.output.jdbc;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import com.google.common.base.Optional;
|
5
|
-
import com.google.common.collect.ImmutableList;
|
6
|
-
import com.fasterxml.jackson.annotation.JsonCreator;
|
7
|
-
import com.fasterxml.jackson.annotation.JsonValue;
|
8
|
-
|
9
|
-
public class JdbcSchema
|
10
|
-
{
|
11
|
-
private List<JdbcColumn> columns;
|
12
|
-
|
13
|
-
@JsonCreator
|
14
|
-
public JdbcSchema(List<JdbcColumn> columns)
|
15
|
-
{
|
16
|
-
this.columns = columns;
|
17
|
-
}
|
18
|
-
|
19
|
-
@JsonValue
|
20
|
-
public List<JdbcColumn> getColumns()
|
21
|
-
{
|
22
|
-
return columns;
|
23
|
-
}
|
24
|
-
|
25
|
-
public Optional<JdbcColumn> findColumn(String name)
|
26
|
-
{
|
27
|
-
for (JdbcColumn column : columns) {
|
28
|
-
if (column.getName().equals(name)) {
|
29
|
-
return Optional.of(column);
|
30
|
-
}
|
31
|
-
}
|
32
|
-
return Optional.absent();
|
33
|
-
}
|
34
|
-
|
35
|
-
public int getCount()
|
36
|
-
{
|
37
|
-
return columns.size();
|
38
|
-
}
|
39
|
-
|
40
|
-
public JdbcColumn getColumn(int i)
|
41
|
-
{
|
42
|
-
return columns.get(i);
|
43
|
-
}
|
44
|
-
|
45
|
-
public String getColumnName(int i)
|
46
|
-
{
|
47
|
-
return columns.get(i).getName();
|
48
|
-
}
|
49
|
-
|
50
|
-
public static JdbcSchema filterSkipColumns(JdbcSchema schema)
|
51
|
-
{
|
52
|
-
ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
|
53
|
-
for (JdbcColumn c : schema.getColumns()) {
|
54
|
-
if (!c.isSkipColumn()) {
|
55
|
-
builder.add(c);
|
56
|
-
}
|
57
|
-
}
|
58
|
-
return new JdbcSchema(builder.build());
|
59
|
-
}
|
60
|
-
}
|
1
|
+
package org.embulk.output.jdbc;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.google.common.collect.ImmutableList;
|
6
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
7
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
8
|
+
|
9
|
+
public class JdbcSchema
|
10
|
+
{
|
11
|
+
private List<JdbcColumn> columns;
|
12
|
+
|
13
|
+
@JsonCreator
|
14
|
+
public JdbcSchema(List<JdbcColumn> columns)
|
15
|
+
{
|
16
|
+
this.columns = columns;
|
17
|
+
}
|
18
|
+
|
19
|
+
@JsonValue
|
20
|
+
public List<JdbcColumn> getColumns()
|
21
|
+
{
|
22
|
+
return columns;
|
23
|
+
}
|
24
|
+
|
25
|
+
public Optional<JdbcColumn> findColumn(String name)
|
26
|
+
{
|
27
|
+
for (JdbcColumn column : columns) {
|
28
|
+
if (column.getName().equals(name)) {
|
29
|
+
return Optional.of(column);
|
30
|
+
}
|
31
|
+
}
|
32
|
+
return Optional.absent();
|
33
|
+
}
|
34
|
+
|
35
|
+
public int getCount()
|
36
|
+
{
|
37
|
+
return columns.size();
|
38
|
+
}
|
39
|
+
|
40
|
+
public JdbcColumn getColumn(int i)
|
41
|
+
{
|
42
|
+
return columns.get(i);
|
43
|
+
}
|
44
|
+
|
45
|
+
public String getColumnName(int i)
|
46
|
+
{
|
47
|
+
return columns.get(i).getName();
|
48
|
+
}
|
49
|
+
|
50
|
+
public static JdbcSchema filterSkipColumns(JdbcSchema schema)
|
51
|
+
{
|
52
|
+
ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
|
53
|
+
for (JdbcColumn c : schema.getColumns()) {
|
54
|
+
if (!c.isSkipColumn()) {
|
55
|
+
builder.add(c);
|
56
|
+
}
|
57
|
+
}
|
58
|
+
return new JdbcSchema(builder.build());
|
59
|
+
}
|
60
|
+
}
|
@@ -1,155 +1,153 @@
|
|
1
|
-
package org.embulk.output.jdbc;
|
2
|
-
|
3
|
-
import java.sql.
|
4
|
-
import java.sql.
|
5
|
-
import java.
|
6
|
-
import java.
|
7
|
-
import
|
8
|
-
import
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
public
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
//
|
39
|
-
|
40
|
-
//
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
logger.info(String.format("> %.2f seconds", seconds));
|
59
|
-
}
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
//
|
66
|
-
//
|
67
|
-
//
|
68
|
-
//
|
69
|
-
//
|
70
|
-
//
|
71
|
-
//
|
72
|
-
// }
|
73
|
-
//
|
74
|
-
|
75
|
-
//
|
76
|
-
|
77
|
-
//
|
78
|
-
//
|
79
|
-
//
|
80
|
-
//
|
81
|
-
//
|
82
|
-
//
|
83
|
-
//
|
84
|
-
//
|
85
|
-
//
|
86
|
-
//
|
87
|
-
//
|
88
|
-
|
89
|
-
//
|
90
|
-
|
91
|
-
//
|
92
|
-
//
|
93
|
-
//
|
94
|
-
//
|
95
|
-
//
|
96
|
-
//
|
97
|
-
//
|
98
|
-
//
|
99
|
-
//{
|
100
|
-
//
|
101
|
-
//
|
102
|
-
//
|
103
|
-
// return
|
104
|
-
// }
|
105
|
-
//
|
106
|
-
// }
|
107
|
-
//
|
108
|
-
|
109
|
-
//
|
110
|
-
|
111
|
-
//
|
112
|
-
//
|
113
|
-
//
|
114
|
-
//
|
115
|
-
//
|
116
|
-
//
|
117
|
-
//
|
118
|
-
//
|
119
|
-
//
|
120
|
-
//
|
121
|
-
//
|
122
|
-
//
|
123
|
-
//
|
124
|
-
//
|
125
|
-
//
|
126
|
-
//
|
127
|
-
//
|
128
|
-
//
|
129
|
-
//
|
130
|
-
// //
|
131
|
-
// //if (cause instanceof
|
132
|
-
// // throw (
|
133
|
-
// //} else if (cause instanceof
|
134
|
-
// // throw (
|
135
|
-
// //} else
|
136
|
-
// // throw (
|
137
|
-
// //}
|
138
|
-
//
|
139
|
-
// //
|
140
|
-
//
|
141
|
-
//
|
142
|
-
//
|
143
|
-
//
|
144
|
-
//
|
145
|
-
//
|
146
|
-
//
|
147
|
-
//
|
148
|
-
// LOG.warn("Connection.setNetworkTimeout is not available
|
149
|
-
// }
|
150
|
-
//
|
151
|
-
//
|
152
|
-
|
153
|
-
|
154
|
-
}
|
155
|
-
|
1
|
+
package org.embulk.output.jdbc;
|
2
|
+
|
3
|
+
import java.sql.SQLException;
|
4
|
+
import java.sql.Statement;
|
5
|
+
import java.util.regex.Matcher;
|
6
|
+
import java.util.regex.Pattern;
|
7
|
+
import org.slf4j.Logger;
|
8
|
+
import org.embulk.spi.Exec;
|
9
|
+
|
10
|
+
public class JdbcUtils
|
11
|
+
{
|
12
|
+
public final Logger logger = Exec.getLogger(JdbcUtils.class.getName());
|
13
|
+
|
14
|
+
private static String[] SEARCH_STRING_SPECIAL_CHARS = new String[] { "_", "%" };
|
15
|
+
|
16
|
+
public static String escapeSearchString(String searchString, String escapeString)
|
17
|
+
{
|
18
|
+
if (searchString != null && escapeString != null) {
|
19
|
+
// First of all, escape escapeString '\' itself: '\' -> '\\'
|
20
|
+
searchString = searchString.replaceAll(Pattern.quote(escapeString),
|
21
|
+
Matcher.quoteReplacement(escapeString + escapeString));
|
22
|
+
for (String specialChar : SEARCH_STRING_SPECIAL_CHARS) {
|
23
|
+
if (specialChar.equals(escapeString)) {
|
24
|
+
throw new IllegalArgumentException("Special char " + specialChar + " cannot be an escape char");
|
25
|
+
}
|
26
|
+
searchString = searchString.replaceAll(Pattern.quote(specialChar),
|
27
|
+
Matcher.quoteReplacement(escapeString + specialChar));
|
28
|
+
}
|
29
|
+
}
|
30
|
+
return searchString;
|
31
|
+
}
|
32
|
+
|
33
|
+
private Class<?> connectionClass;
|
34
|
+
|
35
|
+
// Connection.isValid() is available from Java 1.6 + JDBC4.
|
36
|
+
//private boolean hasIsValid;
|
37
|
+
|
38
|
+
// Connection.setNetworkTimeout() is available from Java 1.7 + JDBC4.
|
39
|
+
//private boolean hasSetNetworkTimeout;
|
40
|
+
//private Method setNetworkTimeoutMethod;
|
41
|
+
|
42
|
+
public JdbcUtils(Class<?> connectionClass)
|
43
|
+
{
|
44
|
+
this.connectionClass = connectionClass;
|
45
|
+
|
46
|
+
//configureSetNetworkTimeout();
|
47
|
+
}
|
48
|
+
|
49
|
+
public int executeUpdateWithSqlLogging(Statement stmt, String sql) throws SQLException
|
50
|
+
{
|
51
|
+
logger.info("SQL: " + sql);
|
52
|
+
long startTime = System.currentTimeMillis();
|
53
|
+
int count = stmt.executeUpdate(sql);
|
54
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
55
|
+
if (count == 0) {
|
56
|
+
logger.info(String.format("> %.2f seconds", seconds));
|
57
|
+
} else {
|
58
|
+
logger.info(String.format("> %.2f seconds (%,d rows)", seconds, count));
|
59
|
+
}
|
60
|
+
return count;
|
61
|
+
}
|
62
|
+
|
63
|
+
//private void configureSetNetworkTimeout() {
|
64
|
+
// try {
|
65
|
+
// Method m = connectionClass.getMethod("setNetworkTimeout", Executor.class, int.class);
|
66
|
+
// if (isCallableMethod(m)) {
|
67
|
+
// setNetworkTimeoutMethod = m;
|
68
|
+
// hasSetNetworkTimeout = true;
|
69
|
+
// }
|
70
|
+
// } catch (SecurityException ex) {
|
71
|
+
// } catch (NoSuchMethodException ex) {
|
72
|
+
// }
|
73
|
+
//}
|
74
|
+
|
75
|
+
//private boolean isCallableMethod(Method m) {
|
76
|
+
// int modifiers = m.getModifiers();
|
77
|
+
// if (Modifier.isAbstract(modifiers)) {
|
78
|
+
// // Method.invoke throws java.lang.AbstractMethodError if it's an
|
79
|
+
// // abstract method. Applications can't catch AbstractMethodError.
|
80
|
+
// return false;
|
81
|
+
// }
|
82
|
+
// if (!Modifier.isPublic(modifiers)) {
|
83
|
+
// // we can only call public methods
|
84
|
+
// return false;
|
85
|
+
// }
|
86
|
+
// return true;
|
87
|
+
//}
|
88
|
+
|
89
|
+
// PostgreSQL JDBC driver implements isValid() method. But the
|
90
|
+
// implementation throws following exception:
|
91
|
+
// "java.io.IOException: Method org.postgresql.jdbc4.Jdbc4Connection.isValid(int) is not yet implemented."
|
92
|
+
//
|
93
|
+
// So, checking mechanism doesn't work at all.
|
94
|
+
// Thus here just runs "SELECT 1" to check connectivity.
|
95
|
+
//
|
96
|
+
//public boolean isValidConnection(Connection connection, int timeout) throws SQLException
|
97
|
+
//{
|
98
|
+
// Statement stmt = connection.createStatement();
|
99
|
+
// try {
|
100
|
+
// stmt.executeQuery("SELECT 1").close();
|
101
|
+
// return true;
|
102
|
+
// } catch (SQLException ex) {
|
103
|
+
// return false;
|
104
|
+
// } finally {
|
105
|
+
// stmt.close();
|
106
|
+
// }
|
107
|
+
//}
|
108
|
+
|
109
|
+
//public void setNetworkTimeout(Connection connection,
|
110
|
+
// Executor executor, int milliseconds) throws SQLException {
|
111
|
+
// Throwable exception = null;
|
112
|
+
// if (hasSetNetworkTimeout) {
|
113
|
+
// try {
|
114
|
+
// setNetworkTimeoutMethod.invoke(connection, executor, milliseconds);
|
115
|
+
// return;
|
116
|
+
//
|
117
|
+
// } catch (IllegalArgumentException ex) {
|
118
|
+
// // ignore error
|
119
|
+
// LOG.warn("Connection.setNetworkTimeout failed due to IllegalArgumentException.");
|
120
|
+
// exception = ex;
|
121
|
+
//
|
122
|
+
// } catch (IllegalAccessException ex) {
|
123
|
+
// // ignore error
|
124
|
+
// LOG.warn("Connection.setNetworkTimeout failed due to IllegalAccessException.");
|
125
|
+
// exception = ex;
|
126
|
+
//
|
127
|
+
// } catch (InvocationTargetException ex) {
|
128
|
+
// //Throwable cause = ex.getTargetException();
|
129
|
+
// //if (cause instanceof SQLException) {
|
130
|
+
// // throw (SQLException) cause;
|
131
|
+
// //} else if (cause instanceof RuntimeException) {
|
132
|
+
// // throw (RuntimeException) cause;
|
133
|
+
// //} else if (cause instanceof Error) {
|
134
|
+
// // throw (Error) cause;
|
135
|
+
// //} else {
|
136
|
+
// // throw new SQLException(cause);
|
137
|
+
// //}
|
138
|
+
// exception = ex.getTargetException();
|
139
|
+
// // It's safe to ignore exceptions.
|
140
|
+
// }
|
141
|
+
//
|
142
|
+
// hasSetNetworkTimeout = false;
|
143
|
+
// }
|
144
|
+
//
|
145
|
+
// if (exception != null) {
|
146
|
+
// LOG.warn("Connection.setNetworkTimeout is not available: "+exception);
|
147
|
+
// } else {
|
148
|
+
// LOG.warn("Connection.setNetworkTimeout is not available.");
|
149
|
+
// }
|
150
|
+
// // TODO any substitute implementations?
|
151
|
+
//}
|
152
|
+
}
|
153
|
+
|
@@ -1,200 +1,200 @@
|
|
1
|
-
package org.embulk.output.jdbc;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import java.util.Calendar;
|
5
|
-
import java.io.IOException;
|
6
|
-
import java.math.BigDecimal;
|
7
|
-
import java.sql.PreparedStatement;
|
8
|
-
import java.sql.SQLException;
|
9
|
-
import java.sql.Date;
|
10
|
-
import java.sql.Time;
|
11
|
-
import com.google.common.base.Optional;
|
12
|
-
import org.slf4j.Logger;
|
13
|
-
import org.embulk.spi.time.Timestamp;
|
14
|
-
import org.embulk.spi.Exec;
|
15
|
-
|
16
|
-
public class StandardBatchInsert
|
17
|
-
implements BatchInsert
|
18
|
-
{
|
19
|
-
private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
|
20
|
-
|
21
|
-
private final JdbcOutputConnector connector;
|
22
|
-
private final Optional<List<String>> mergeKeys;
|
23
|
-
|
24
|
-
private JdbcOutputConnection connection;
|
25
|
-
private PreparedStatement batch;
|
26
|
-
private int index;
|
27
|
-
private int batchWeight;
|
28
|
-
private int batchRows;
|
29
|
-
private long totalRows;
|
30
|
-
|
31
|
-
public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
32
|
-
{
|
33
|
-
this.connector = connector;
|
34
|
-
this.mergeKeys = mergeKeys;
|
35
|
-
}
|
36
|
-
|
37
|
-
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
38
|
-
{
|
39
|
-
this.connection = connector.connect(true);
|
40
|
-
this.index = 1; // PreparedStatement index begings from 1
|
41
|
-
this.batchRows = 0;
|
42
|
-
this.totalRows = 0;
|
43
|
-
this.batch = prepareStatement(loadTable, insertSchema);
|
44
|
-
batch.clearBatch();
|
45
|
-
}
|
46
|
-
|
47
|
-
protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
|
48
|
-
{
|
49
|
-
return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
|
50
|
-
}
|
51
|
-
|
52
|
-
public int getBatchWeight()
|
53
|
-
{
|
54
|
-
return batchWeight;
|
55
|
-
}
|
56
|
-
|
57
|
-
public void add() throws IOException, SQLException
|
58
|
-
{
|
59
|
-
batch.addBatch();
|
60
|
-
index = 1; // PreparedStatement index begins from 1
|
61
|
-
batchRows++;
|
62
|
-
batchWeight += 32; // add weight as overhead of each rows
|
63
|
-
}
|
64
|
-
|
65
|
-
public void close() throws IOException, SQLException
|
66
|
-
{
|
67
|
-
// caller should close the connection
|
68
|
-
}
|
69
|
-
|
70
|
-
public void flush() throws IOException, SQLException
|
71
|
-
{
|
72
|
-
logger.info(String.format("Loading %,d rows", batchRows));
|
73
|
-
long startTime = System.currentTimeMillis();
|
74
|
-
batch.executeBatch(); // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
|
75
|
-
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
76
|
-
|
77
|
-
totalRows += batchRows;
|
78
|
-
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
79
|
-
batch.clearBatch();
|
80
|
-
batchRows = 0;
|
81
|
-
batchWeight = 0;
|
82
|
-
}
|
83
|
-
|
84
|
-
public void finish() throws IOException, SQLException
|
85
|
-
{
|
86
|
-
if (getBatchWeight() != 0) {
|
87
|
-
flush();
|
88
|
-
}
|
89
|
-
}
|
90
|
-
|
91
|
-
public void setNull(int sqlType) throws IOException, SQLException
|
92
|
-
{
|
93
|
-
batch.setNull(index, sqlType);
|
94
|
-
nextColumn(0);
|
95
|
-
}
|
96
|
-
|
97
|
-
public void setBoolean(boolean v) throws IOException, SQLException
|
98
|
-
{
|
99
|
-
batch.setBoolean(index, v);
|
100
|
-
nextColumn(1);
|
101
|
-
}
|
102
|
-
|
103
|
-
public void setByte(byte v) throws IOException, SQLException
|
104
|
-
{
|
105
|
-
batch.setByte(index, v);
|
106
|
-
nextColumn(1);
|
107
|
-
}
|
108
|
-
|
109
|
-
public void setShort(short v) throws IOException, SQLException
|
110
|
-
{
|
111
|
-
batch.setShort(index, v);
|
112
|
-
nextColumn(2);
|
113
|
-
}
|
114
|
-
|
115
|
-
public void setInt(int v) throws IOException, SQLException
|
116
|
-
{
|
117
|
-
batch.setInt(index, v);
|
118
|
-
nextColumn(4);
|
119
|
-
}
|
120
|
-
|
121
|
-
public void setLong(long v) throws IOException, SQLException
|
122
|
-
{
|
123
|
-
batch.setLong(index, v);
|
124
|
-
nextColumn(8);
|
125
|
-
}
|
126
|
-
|
127
|
-
public void setFloat(float v) throws IOException, SQLException
|
128
|
-
{
|
129
|
-
batch.setFloat(index, v);
|
130
|
-
nextColumn(4);
|
131
|
-
}
|
132
|
-
|
133
|
-
public void setDouble(double v) throws IOException, SQLException
|
134
|
-
{
|
135
|
-
batch.setDouble(index, v);
|
136
|
-
nextColumn(8);
|
137
|
-
}
|
138
|
-
|
139
|
-
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
140
|
-
{
|
141
|
-
// use estimated number of necessary bytes + 8 byte for the weight
|
142
|
-
// assuming one place needs 4 bits. ceil(v.precision() / 2.0) + 8
|
143
|
-
batch.setBigDecimal(index, v);
|
144
|
-
nextColumn((v.precision() & ~2) / 2 + 8);
|
145
|
-
}
|
146
|
-
|
147
|
-
public void setString(String v) throws IOException, SQLException
|
148
|
-
{
|
149
|
-
batch.setString(index, v);
|
150
|
-
// estimate all chracters use 2 bytes; almost enough for the worst case
|
151
|
-
nextColumn(v.length() * 2 + 4);
|
152
|
-
}
|
153
|
-
|
154
|
-
public void setNString(String v) throws IOException, SQLException
|
155
|
-
{
|
156
|
-
batch.setNString(index, v);
|
157
|
-
// estimate all chracters use 2 bytes; almost enough for the worst case
|
158
|
-
nextColumn(v.length() * 2 + 4);
|
159
|
-
}
|
160
|
-
|
161
|
-
public void setBytes(byte[] v) throws IOException, SQLException
|
162
|
-
{
|
163
|
-
batch.setBytes(index, v);
|
164
|
-
nextColumn(v.length + 4);
|
165
|
-
}
|
166
|
-
|
167
|
-
public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
|
168
|
-
{
|
169
|
-
// JavaDoc of java.sql.Time says:
|
170
|
-
// >> To conform with the definition of SQL DATE, the millisecond values wrapped by a java.sql.Date instance must be 'normalized' by setting the hours, minutes, seconds, and milliseconds to zero in the particular time zone with which the instance is associated.
|
171
|
-
cal.setTimeInMillis(v.getEpochSecond() * 1000);
|
172
|
-
cal.set(Calendar.SECOND, 0);
|
173
|
-
cal.set(Calendar.MINUTE, 0);
|
174
|
-
cal.set(Calendar.HOUR_OF_DAY, 0);
|
175
|
-
Date normalized = new Date(cal.getTimeInMillis());
|
176
|
-
batch.setDate(index, normalized, cal);
|
177
|
-
nextColumn(32);
|
178
|
-
}
|
179
|
-
|
180
|
-
public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
|
181
|
-
{
|
182
|
-
Time t = new Time(v.toEpochMilli());
|
183
|
-
batch.setTime(index, t, cal);
|
184
|
-
nextColumn(32);
|
185
|
-
}
|
186
|
-
|
187
|
-
public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
|
188
|
-
{
|
189
|
-
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
190
|
-
t.setNanos(v.getNano());
|
191
|
-
batch.setTimestamp(index, t, cal);
|
192
|
-
nextColumn(32);
|
193
|
-
}
|
194
|
-
|
195
|
-
private void nextColumn(int weight)
|
196
|
-
{
|
197
|
-
index++;
|
198
|
-
batchWeight += weight + 4; // add weight as overhead of each columns
|
199
|
-
}
|
200
|
-
}
|
1
|
+
package org.embulk.output.jdbc;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Calendar;
|
5
|
+
import java.io.IOException;
|
6
|
+
import java.math.BigDecimal;
|
7
|
+
import java.sql.PreparedStatement;
|
8
|
+
import java.sql.SQLException;
|
9
|
+
import java.sql.Date;
|
10
|
+
import java.sql.Time;
|
11
|
+
import com.google.common.base.Optional;
|
12
|
+
import org.slf4j.Logger;
|
13
|
+
import org.embulk.spi.time.Timestamp;
|
14
|
+
import org.embulk.spi.Exec;
|
15
|
+
|
16
|
+
public class StandardBatchInsert
|
17
|
+
implements BatchInsert
|
18
|
+
{
|
19
|
+
private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
|
20
|
+
|
21
|
+
private final JdbcOutputConnector connector;
|
22
|
+
private final Optional<List<String>> mergeKeys;
|
23
|
+
|
24
|
+
private JdbcOutputConnection connection;
|
25
|
+
private PreparedStatement batch;
|
26
|
+
private int index;
|
27
|
+
private int batchWeight;
|
28
|
+
private int batchRows;
|
29
|
+
private long totalRows;
|
30
|
+
|
31
|
+
public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
32
|
+
{
|
33
|
+
this.connector = connector;
|
34
|
+
this.mergeKeys = mergeKeys;
|
35
|
+
}
|
36
|
+
|
37
|
+
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
38
|
+
{
|
39
|
+
this.connection = connector.connect(true);
|
40
|
+
this.index = 1; // PreparedStatement index begings from 1
|
41
|
+
this.batchRows = 0;
|
42
|
+
this.totalRows = 0;
|
43
|
+
this.batch = prepareStatement(loadTable, insertSchema);
|
44
|
+
batch.clearBatch();
|
45
|
+
}
|
46
|
+
|
47
|
+
protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
|
48
|
+
{
|
49
|
+
return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
|
50
|
+
}
|
51
|
+
|
52
|
+
public int getBatchWeight()
|
53
|
+
{
|
54
|
+
return batchWeight;
|
55
|
+
}
|
56
|
+
|
57
|
+
public void add() throws IOException, SQLException
|
58
|
+
{
|
59
|
+
batch.addBatch();
|
60
|
+
index = 1; // PreparedStatement index begins from 1
|
61
|
+
batchRows++;
|
62
|
+
batchWeight += 32; // add weight as overhead of each rows
|
63
|
+
}
|
64
|
+
|
65
|
+
public void close() throws IOException, SQLException
|
66
|
+
{
|
67
|
+
// caller should close the connection
|
68
|
+
}
|
69
|
+
|
70
|
+
public void flush() throws IOException, SQLException
|
71
|
+
{
|
72
|
+
logger.info(String.format("Loading %,d rows", batchRows));
|
73
|
+
long startTime = System.currentTimeMillis();
|
74
|
+
batch.executeBatch(); // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
|
75
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
76
|
+
|
77
|
+
totalRows += batchRows;
|
78
|
+
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
79
|
+
batch.clearBatch();
|
80
|
+
batchRows = 0;
|
81
|
+
batchWeight = 0;
|
82
|
+
}
|
83
|
+
|
84
|
+
public void finish() throws IOException, SQLException
|
85
|
+
{
|
86
|
+
if (getBatchWeight() != 0) {
|
87
|
+
flush();
|
88
|
+
}
|
89
|
+
}
|
90
|
+
|
91
|
+
public void setNull(int sqlType) throws IOException, SQLException
|
92
|
+
{
|
93
|
+
batch.setNull(index, sqlType);
|
94
|
+
nextColumn(0);
|
95
|
+
}
|
96
|
+
|
97
|
+
public void setBoolean(boolean v) throws IOException, SQLException
|
98
|
+
{
|
99
|
+
batch.setBoolean(index, v);
|
100
|
+
nextColumn(1);
|
101
|
+
}
|
102
|
+
|
103
|
+
public void setByte(byte v) throws IOException, SQLException
|
104
|
+
{
|
105
|
+
batch.setByte(index, v);
|
106
|
+
nextColumn(1);
|
107
|
+
}
|
108
|
+
|
109
|
+
public void setShort(short v) throws IOException, SQLException
|
110
|
+
{
|
111
|
+
batch.setShort(index, v);
|
112
|
+
nextColumn(2);
|
113
|
+
}
|
114
|
+
|
115
|
+
public void setInt(int v) throws IOException, SQLException
|
116
|
+
{
|
117
|
+
batch.setInt(index, v);
|
118
|
+
nextColumn(4);
|
119
|
+
}
|
120
|
+
|
121
|
+
public void setLong(long v) throws IOException, SQLException
|
122
|
+
{
|
123
|
+
batch.setLong(index, v);
|
124
|
+
nextColumn(8);
|
125
|
+
}
|
126
|
+
|
127
|
+
public void setFloat(float v) throws IOException, SQLException
|
128
|
+
{
|
129
|
+
batch.setFloat(index, v);
|
130
|
+
nextColumn(4);
|
131
|
+
}
|
132
|
+
|
133
|
+
public void setDouble(double v) throws IOException, SQLException
|
134
|
+
{
|
135
|
+
batch.setDouble(index, v);
|
136
|
+
nextColumn(8);
|
137
|
+
}
|
138
|
+
|
139
|
+
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
140
|
+
{
|
141
|
+
// use estimated number of necessary bytes + 8 byte for the weight
|
142
|
+
// assuming one place needs 4 bits. ceil(v.precision() / 2.0) + 8
|
143
|
+
batch.setBigDecimal(index, v);
|
144
|
+
nextColumn((v.precision() & ~2) / 2 + 8);
|
145
|
+
}
|
146
|
+
|
147
|
+
public void setString(String v) throws IOException, SQLException
|
148
|
+
{
|
149
|
+
batch.setString(index, v);
|
150
|
+
// estimate all chracters use 2 bytes; almost enough for the worst case
|
151
|
+
nextColumn(v.length() * 2 + 4);
|
152
|
+
}
|
153
|
+
|
154
|
+
public void setNString(String v) throws IOException, SQLException
|
155
|
+
{
|
156
|
+
batch.setNString(index, v);
|
157
|
+
// estimate all chracters use 2 bytes; almost enough for the worst case
|
158
|
+
nextColumn(v.length() * 2 + 4);
|
159
|
+
}
|
160
|
+
|
161
|
+
public void setBytes(byte[] v) throws IOException, SQLException
|
162
|
+
{
|
163
|
+
batch.setBytes(index, v);
|
164
|
+
nextColumn(v.length + 4);
|
165
|
+
}
|
166
|
+
|
167
|
+
public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
|
168
|
+
{
|
169
|
+
// JavaDoc of java.sql.Time says:
|
170
|
+
// >> To conform with the definition of SQL DATE, the millisecond values wrapped by a java.sql.Date instance must be 'normalized' by setting the hours, minutes, seconds, and milliseconds to zero in the particular time zone with which the instance is associated.
|
171
|
+
cal.setTimeInMillis(v.getEpochSecond() * 1000);
|
172
|
+
cal.set(Calendar.SECOND, 0);
|
173
|
+
cal.set(Calendar.MINUTE, 0);
|
174
|
+
cal.set(Calendar.HOUR_OF_DAY, 0);
|
175
|
+
Date normalized = new Date(cal.getTimeInMillis());
|
176
|
+
batch.setDate(index, normalized, cal);
|
177
|
+
nextColumn(32);
|
178
|
+
}
|
179
|
+
|
180
|
+
public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
|
181
|
+
{
|
182
|
+
Time t = new Time(v.toEpochMilli());
|
183
|
+
batch.setTime(index, t, cal);
|
184
|
+
nextColumn(32);
|
185
|
+
}
|
186
|
+
|
187
|
+
public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
|
188
|
+
{
|
189
|
+
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
190
|
+
t.setNanos(v.getNano());
|
191
|
+
batch.setTimestamp(index, t, cal);
|
192
|
+
nextColumn(32);
|
193
|
+
}
|
194
|
+
|
195
|
+
private void nextColumn(int weight)
|
196
|
+
{
|
197
|
+
index++;
|
198
|
+
batchWeight += weight + 4; // add weight as overhead of each columns
|
199
|
+
}
|
200
|
+
}
|