embulk-output-jdbc 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/build.gradle +2 -2
  3. data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
  4. data/lib/embulk/output/jdbc.rb +3 -3
  5. data/src/main/java/org/embulk/output/JdbcOutputPlugin.java +137 -138
  6. data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +972 -973
  7. data/src/main/java/org/embulk/output/jdbc/BatchInsert.java +52 -53
  8. data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java +116 -116
  9. data/src/main/java/org/embulk/output/jdbc/JdbcColumnOption.java +34 -34
  10. data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +492 -492
  11. data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java +8 -8
  12. data/src/main/java/org/embulk/output/jdbc/JdbcSchema.java +60 -60
  13. data/src/main/java/org/embulk/output/jdbc/JdbcUtils.java +153 -155
  14. data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +200 -200
  15. data/src/main/java/org/embulk/output/jdbc/ToString.java +54 -54
  16. data/src/main/java/org/embulk/output/jdbc/ToStringMap.java +34 -35
  17. data/src/main/java/org/embulk/output/jdbc/setter/BigDecimalColumnSetter.java +68 -70
  18. data/src/main/java/org/embulk/output/jdbc/setter/BooleanColumnSetter.java +53 -54
  19. data/src/main/java/org/embulk/output/jdbc/setter/ByteColumnSetter.java +75 -76
  20. data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java +44 -45
  21. data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterFactory.java +196 -196
  22. data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterVisitor.java +95 -96
  23. data/src/main/java/org/embulk/output/jdbc/setter/DefaultValueSetter.java +48 -48
  24. data/src/main/java/org/embulk/output/jdbc/setter/DoubleColumnSetter.java +60 -61
  25. data/src/main/java/org/embulk/output/jdbc/setter/FloatColumnSetter.java +60 -61
  26. data/src/main/java/org/embulk/output/jdbc/setter/IntColumnSetter.java +75 -76
  27. data/src/main/java/org/embulk/output/jdbc/setter/LongColumnSetter.java +71 -72
  28. data/src/main/java/org/embulk/output/jdbc/setter/NStringColumnSetter.java +58 -59
  29. data/src/main/java/org/embulk/output/jdbc/setter/NullColumnSetter.java +53 -53
  30. data/src/main/java/org/embulk/output/jdbc/setter/NullDefaultValueSetter.java +105 -105
  31. data/src/main/java/org/embulk/output/jdbc/setter/PassThroughColumnSetter.java +58 -59
  32. data/src/main/java/org/embulk/output/jdbc/setter/ShortColumnSetter.java +75 -76
  33. data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java +43 -43
  34. data/src/main/java/org/embulk/output/jdbc/setter/SqlDateColumnSetter.java +58 -59
  35. data/src/main/java/org/embulk/output/jdbc/setter/SqlTimeColumnSetter.java +58 -59
  36. data/src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java +58 -58
  37. data/src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java +58 -59
  38. data/src/test/java/org/embulk/output/TestJdbcOutputPlugin.java +5 -5
  39. metadata +3 -3
  40. data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
@@ -1,8 +1,8 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.sql.SQLException;
4
-
5
- public interface JdbcOutputConnector
6
- {
7
- public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
8
- }
1
+ package org.embulk.output.jdbc;
2
+
3
+ import java.sql.SQLException;
4
+
5
+ public interface JdbcOutputConnector
6
+ {
7
+ public JdbcOutputConnection connect(boolean autoCommit) throws SQLException;
8
+ }
@@ -1,60 +1,60 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.util.List;
4
- import com.google.common.base.Optional;
5
- import com.google.common.collect.ImmutableList;
6
- import com.fasterxml.jackson.annotation.JsonCreator;
7
- import com.fasterxml.jackson.annotation.JsonValue;
8
-
9
- public class JdbcSchema
10
- {
11
- private List<JdbcColumn> columns;
12
-
13
- @JsonCreator
14
- public JdbcSchema(List<JdbcColumn> columns)
15
- {
16
- this.columns = columns;
17
- }
18
-
19
- @JsonValue
20
- public List<JdbcColumn> getColumns()
21
- {
22
- return columns;
23
- }
24
-
25
- public Optional<JdbcColumn> findColumn(String name)
26
- {
27
- for (JdbcColumn column : columns) {
28
- if (column.getName().equals(name)) {
29
- return Optional.of(column);
30
- }
31
- }
32
- return Optional.absent();
33
- }
34
-
35
- public int getCount()
36
- {
37
- return columns.size();
38
- }
39
-
40
- public JdbcColumn getColumn(int i)
41
- {
42
- return columns.get(i);
43
- }
44
-
45
- public String getColumnName(int i)
46
- {
47
- return columns.get(i).getName();
48
- }
49
-
50
- public static JdbcSchema filterSkipColumns(JdbcSchema schema)
51
- {
52
- ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
53
- for (JdbcColumn c : schema.getColumns()) {
54
- if (!c.isSkipColumn()) {
55
- builder.add(c);
56
- }
57
- }
58
- return new JdbcSchema(builder.build());
59
- }
60
- }
1
+ package org.embulk.output.jdbc;
2
+
3
+ import java.util.List;
4
+ import com.google.common.base.Optional;
5
+ import com.google.common.collect.ImmutableList;
6
+ import com.fasterxml.jackson.annotation.JsonCreator;
7
+ import com.fasterxml.jackson.annotation.JsonValue;
8
+
9
+ public class JdbcSchema
10
+ {
11
+ private List<JdbcColumn> columns;
12
+
13
+ @JsonCreator
14
+ public JdbcSchema(List<JdbcColumn> columns)
15
+ {
16
+ this.columns = columns;
17
+ }
18
+
19
+ @JsonValue
20
+ public List<JdbcColumn> getColumns()
21
+ {
22
+ return columns;
23
+ }
24
+
25
+ public Optional<JdbcColumn> findColumn(String name)
26
+ {
27
+ for (JdbcColumn column : columns) {
28
+ if (column.getName().equals(name)) {
29
+ return Optional.of(column);
30
+ }
31
+ }
32
+ return Optional.absent();
33
+ }
34
+
35
+ public int getCount()
36
+ {
37
+ return columns.size();
38
+ }
39
+
40
+ public JdbcColumn getColumn(int i)
41
+ {
42
+ return columns.get(i);
43
+ }
44
+
45
+ public String getColumnName(int i)
46
+ {
47
+ return columns.get(i).getName();
48
+ }
49
+
50
+ public static JdbcSchema filterSkipColumns(JdbcSchema schema)
51
+ {
52
+ ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
53
+ for (JdbcColumn c : schema.getColumns()) {
54
+ if (!c.isSkipColumn()) {
55
+ builder.add(c);
56
+ }
57
+ }
58
+ return new JdbcSchema(builder.build());
59
+ }
60
+ }
@@ -1,155 +1,153 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.sql.Connection;
4
- import java.sql.PreparedStatement;
5
- import java.sql.SQLException;
6
- import java.sql.Statement;
7
- import java.util.regex.Matcher;
8
- import java.util.regex.Pattern;
9
- import org.slf4j.Logger;
10
- import org.embulk.spi.Exec;
11
-
12
- public class JdbcUtils
13
- {
14
- public final Logger logger = Exec.getLogger(JdbcUtils.class.getName());
15
-
16
- private static String[] SEARCH_STRING_SPECIAL_CHARS = new String[] { "_", "%" };
17
-
18
- public static String escapeSearchString(String searchString, String escapeString)
19
- {
20
- if (searchString != null && escapeString != null) {
21
- // First of all, escape escapeString '\' itself: '\' -> '\\'
22
- searchString = searchString.replaceAll(Pattern.quote(escapeString),
23
- Matcher.quoteReplacement(escapeString + escapeString));
24
- for (String specialChar : SEARCH_STRING_SPECIAL_CHARS) {
25
- if (specialChar.equals(escapeString)) {
26
- throw new IllegalArgumentException("Special char " + specialChar + " cannot be an escape char");
27
- }
28
- searchString = searchString.replaceAll(Pattern.quote(specialChar),
29
- Matcher.quoteReplacement(escapeString + specialChar));
30
- }
31
- }
32
- return searchString;
33
- }
34
-
35
- private Class<?> connectionClass;
36
-
37
- // Connection.isValid() is available from Java 1.6 + JDBC4.
38
- //private boolean hasIsValid;
39
-
40
- // Connection.setNetworkTimeout() is available from Java 1.7 + JDBC4.
41
- //private boolean hasSetNetworkTimeout;
42
- //private Method setNetworkTimeoutMethod;
43
-
44
- public JdbcUtils(Class<?> connectionClass)
45
- {
46
- this.connectionClass = connectionClass;
47
-
48
- //configureSetNetworkTimeout();
49
- }
50
-
51
- public int executeUpdateWithSqlLogging(Statement stmt, String sql) throws SQLException
52
- {
53
- logger.info("SQL: " + sql);
54
- long startTime = System.currentTimeMillis();
55
- int count = stmt.executeUpdate(sql);
56
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
57
- if (count == 0) {
58
- logger.info(String.format("> %.2f seconds", seconds));
59
- } else {
60
- logger.info(String.format("> %.2f seconds (%,d rows)", seconds, count));
61
- }
62
- return count;
63
- }
64
-
65
- //private void configureSetNetworkTimeout() {
66
- // try {
67
- // Method m = connectionClass.getMethod("setNetworkTimeout", Executor.class, int.class);
68
- // if (isCallableMethod(m)) {
69
- // setNetworkTimeoutMethod = m;
70
- // hasSetNetworkTimeout = true;
71
- // }
72
- // } catch (SecurityException ex) {
73
- // } catch (NoSuchMethodException ex) {
74
- // }
75
- //}
76
-
77
- //private boolean isCallableMethod(Method m) {
78
- // int modifiers = m.getModifiers();
79
- // if (Modifier.isAbstract(modifiers)) {
80
- // // Method.invoke throws java.lang.AbstractMethodError if it's an
81
- // // abstract method. Applications can't catch AbstractMethodError.
82
- // return false;
83
- // }
84
- // if (!Modifier.isPublic(modifiers)) {
85
- // // we can only call public methods
86
- // return false;
87
- // }
88
- // return true;
89
- //}
90
-
91
- // PostgreSQL JDBC driver implements isValid() method. But the
92
- // implementation throws following exception:
93
- // "java.io.IOException: Method org.postgresql.jdbc4.Jdbc4Connection.isValid(int) is not yet implemented."
94
- //
95
- // So, checking mechanism doesn't work at all.
96
- // Thus here just runs "SELECT 1" to check connectivity.
97
- //
98
- //public boolean isValidConnection(Connection connection, int timeout) throws SQLException
99
- //{
100
- // Statement stmt = connection.createStatement();
101
- // try {
102
- // stmt.executeQuery("SELECT 1").close();
103
- // return true;
104
- // } catch (SQLException ex) {
105
- // return false;
106
- // } finally {
107
- // stmt.close();
108
- // }
109
- //}
110
-
111
- //public void setNetworkTimeout(Connection connection,
112
- // Executor executor, int milliseconds) throws SQLException {
113
- // Throwable exception = null;
114
- // if (hasSetNetworkTimeout) {
115
- // try {
116
- // setNetworkTimeoutMethod.invoke(connection, executor, milliseconds);
117
- // return;
118
- //
119
- // } catch (IllegalArgumentException ex) {
120
- // // ignore error
121
- // LOG.warn("Connection.setNetworkTimeout failed due to IllegalArgumentException.");
122
- // exception = ex;
123
- //
124
- // } catch (IllegalAccessException ex) {
125
- // // ignore error
126
- // LOG.warn("Connection.setNetworkTimeout failed due to IllegalAccessException.");
127
- // exception = ex;
128
- //
129
- // } catch (InvocationTargetException ex) {
130
- // //Throwable cause = ex.getTargetException();
131
- // //if (cause instanceof SQLException) {
132
- // // throw (SQLException) cause;
133
- // //} else if (cause instanceof RuntimeException) {
134
- // // throw (RuntimeException) cause;
135
- // //} else if (cause instanceof Error) {
136
- // // throw (Error) cause;
137
- // //} else {
138
- // // throw new SQLException(cause);
139
- // //}
140
- // exception = ex.getTargetException();
141
- // // It's safe to ignore exceptions.
142
- // }
143
- //
144
- // hasSetNetworkTimeout = false;
145
- // }
146
- //
147
- // if (exception != null) {
148
- // LOG.warn("Connection.setNetworkTimeout is not available: "+exception);
149
- // } else {
150
- // LOG.warn("Connection.setNetworkTimeout is not available.");
151
- // }
152
- // // TODO any substitute implementations?
153
- //}
154
- }
155
-
1
+ package org.embulk.output.jdbc;
2
+
3
+ import java.sql.SQLException;
4
+ import java.sql.Statement;
5
+ import java.util.regex.Matcher;
6
+ import java.util.regex.Pattern;
7
+ import org.slf4j.Logger;
8
+ import org.embulk.spi.Exec;
9
+
10
+ public class JdbcUtils
11
+ {
12
+ public final Logger logger = Exec.getLogger(JdbcUtils.class.getName());
13
+
14
+ private static String[] SEARCH_STRING_SPECIAL_CHARS = new String[] { "_", "%" };
15
+
16
+ public static String escapeSearchString(String searchString, String escapeString)
17
+ {
18
+ if (searchString != null && escapeString != null) {
19
+ // First of all, escape escapeString '\' itself: '\' -> '\\'
20
+ searchString = searchString.replaceAll(Pattern.quote(escapeString),
21
+ Matcher.quoteReplacement(escapeString + escapeString));
22
+ for (String specialChar : SEARCH_STRING_SPECIAL_CHARS) {
23
+ if (specialChar.equals(escapeString)) {
24
+ throw new IllegalArgumentException("Special char " + specialChar + " cannot be an escape char");
25
+ }
26
+ searchString = searchString.replaceAll(Pattern.quote(specialChar),
27
+ Matcher.quoteReplacement(escapeString + specialChar));
28
+ }
29
+ }
30
+ return searchString;
31
+ }
32
+
33
+ private Class<?> connectionClass;
34
+
35
+ // Connection.isValid() is available from Java 1.6 + JDBC4.
36
+ //private boolean hasIsValid;
37
+
38
+ // Connection.setNetworkTimeout() is available from Java 1.7 + JDBC4.
39
+ //private boolean hasSetNetworkTimeout;
40
+ //private Method setNetworkTimeoutMethod;
41
+
42
+ public JdbcUtils(Class<?> connectionClass)
43
+ {
44
+ this.connectionClass = connectionClass;
45
+
46
+ //configureSetNetworkTimeout();
47
+ }
48
+
49
+ public int executeUpdateWithSqlLogging(Statement stmt, String sql) throws SQLException
50
+ {
51
+ logger.info("SQL: " + sql);
52
+ long startTime = System.currentTimeMillis();
53
+ int count = stmt.executeUpdate(sql);
54
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
55
+ if (count == 0) {
56
+ logger.info(String.format("> %.2f seconds", seconds));
57
+ } else {
58
+ logger.info(String.format("> %.2f seconds (%,d rows)", seconds, count));
59
+ }
60
+ return count;
61
+ }
62
+
63
+ //private void configureSetNetworkTimeout() {
64
+ // try {
65
+ // Method m = connectionClass.getMethod("setNetworkTimeout", Executor.class, int.class);
66
+ // if (isCallableMethod(m)) {
67
+ // setNetworkTimeoutMethod = m;
68
+ // hasSetNetworkTimeout = true;
69
+ // }
70
+ // } catch (SecurityException ex) {
71
+ // } catch (NoSuchMethodException ex) {
72
+ // }
73
+ //}
74
+
75
+ //private boolean isCallableMethod(Method m) {
76
+ // int modifiers = m.getModifiers();
77
+ // if (Modifier.isAbstract(modifiers)) {
78
+ // // Method.invoke throws java.lang.AbstractMethodError if it's an
79
+ // // abstract method. Applications can't catch AbstractMethodError.
80
+ // return false;
81
+ // }
82
+ // if (!Modifier.isPublic(modifiers)) {
83
+ // // we can only call public methods
84
+ // return false;
85
+ // }
86
+ // return true;
87
+ //}
88
+
89
+ // PostgreSQL JDBC driver implements isValid() method. But the
90
+ // implementation throws following exception:
91
+ // "java.io.IOException: Method org.postgresql.jdbc4.Jdbc4Connection.isValid(int) is not yet implemented."
92
+ //
93
+ // So, checking mechanism doesn't work at all.
94
+ // Thus here just runs "SELECT 1" to check connectivity.
95
+ //
96
+ //public boolean isValidConnection(Connection connection, int timeout) throws SQLException
97
+ //{
98
+ // Statement stmt = connection.createStatement();
99
+ // try {
100
+ // stmt.executeQuery("SELECT 1").close();
101
+ // return true;
102
+ // } catch (SQLException ex) {
103
+ // return false;
104
+ // } finally {
105
+ // stmt.close();
106
+ // }
107
+ //}
108
+
109
+ //public void setNetworkTimeout(Connection connection,
110
+ // Executor executor, int milliseconds) throws SQLException {
111
+ // Throwable exception = null;
112
+ // if (hasSetNetworkTimeout) {
113
+ // try {
114
+ // setNetworkTimeoutMethod.invoke(connection, executor, milliseconds);
115
+ // return;
116
+ //
117
+ // } catch (IllegalArgumentException ex) {
118
+ // // ignore error
119
+ // LOG.warn("Connection.setNetworkTimeout failed due to IllegalArgumentException.");
120
+ // exception = ex;
121
+ //
122
+ // } catch (IllegalAccessException ex) {
123
+ // // ignore error
124
+ // LOG.warn("Connection.setNetworkTimeout failed due to IllegalAccessException.");
125
+ // exception = ex;
126
+ //
127
+ // } catch (InvocationTargetException ex) {
128
+ // //Throwable cause = ex.getTargetException();
129
+ // //if (cause instanceof SQLException) {
130
+ // // throw (SQLException) cause;
131
+ // //} else if (cause instanceof RuntimeException) {
132
+ // // throw (RuntimeException) cause;
133
+ // //} else if (cause instanceof Error) {
134
+ // // throw (Error) cause;
135
+ // //} else {
136
+ // // throw new SQLException(cause);
137
+ // //}
138
+ // exception = ex.getTargetException();
139
+ // // It's safe to ignore exceptions.
140
+ // }
141
+ //
142
+ // hasSetNetworkTimeout = false;
143
+ // }
144
+ //
145
+ // if (exception != null) {
146
+ // LOG.warn("Connection.setNetworkTimeout is not available: "+exception);
147
+ // } else {
148
+ // LOG.warn("Connection.setNetworkTimeout is not available.");
149
+ // }
150
+ // // TODO any substitute implementations?
151
+ //}
152
+ }
153
+
@@ -1,200 +1,200 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.util.List;
4
- import java.util.Calendar;
5
- import java.io.IOException;
6
- import java.math.BigDecimal;
7
- import java.sql.PreparedStatement;
8
- import java.sql.SQLException;
9
- import java.sql.Date;
10
- import java.sql.Time;
11
- import com.google.common.base.Optional;
12
- import org.slf4j.Logger;
13
- import org.embulk.spi.time.Timestamp;
14
- import org.embulk.spi.Exec;
15
-
16
- public class StandardBatchInsert
17
- implements BatchInsert
18
- {
19
- private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
20
-
21
- private final JdbcOutputConnector connector;
22
- private final Optional<List<String>> mergeKeys;
23
-
24
- private JdbcOutputConnection connection;
25
- private PreparedStatement batch;
26
- private int index;
27
- private int batchWeight;
28
- private int batchRows;
29
- private long totalRows;
30
-
31
- public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
32
- {
33
- this.connector = connector;
34
- this.mergeKeys = mergeKeys;
35
- }
36
-
37
- public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
38
- {
39
- this.connection = connector.connect(true);
40
- this.index = 1; // PreparedStatement index begings from 1
41
- this.batchRows = 0;
42
- this.totalRows = 0;
43
- this.batch = prepareStatement(loadTable, insertSchema);
44
- batch.clearBatch();
45
- }
46
-
47
- protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
48
- {
49
- return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
50
- }
51
-
52
- public int getBatchWeight()
53
- {
54
- return batchWeight;
55
- }
56
-
57
- public void add() throws IOException, SQLException
58
- {
59
- batch.addBatch();
60
- index = 1; // PreparedStatement index begins from 1
61
- batchRows++;
62
- batchWeight += 32; // add weight as overhead of each rows
63
- }
64
-
65
- public void close() throws IOException, SQLException
66
- {
67
- // caller should close the connection
68
- }
69
-
70
- public void flush() throws IOException, SQLException
71
- {
72
- logger.info(String.format("Loading %,d rows", batchRows));
73
- long startTime = System.currentTimeMillis();
74
- batch.executeBatch(); // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
75
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
76
-
77
- totalRows += batchRows;
78
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
79
- batch.clearBatch();
80
- batchRows = 0;
81
- batchWeight = 0;
82
- }
83
-
84
- public void finish() throws IOException, SQLException
85
- {
86
- if (getBatchWeight() != 0) {
87
- flush();
88
- }
89
- }
90
-
91
- public void setNull(int sqlType) throws IOException, SQLException
92
- {
93
- batch.setNull(index, sqlType);
94
- nextColumn(0);
95
- }
96
-
97
- public void setBoolean(boolean v) throws IOException, SQLException
98
- {
99
- batch.setBoolean(index, v);
100
- nextColumn(1);
101
- }
102
-
103
- public void setByte(byte v) throws IOException, SQLException
104
- {
105
- batch.setByte(index, v);
106
- nextColumn(1);
107
- }
108
-
109
- public void setShort(short v) throws IOException, SQLException
110
- {
111
- batch.setShort(index, v);
112
- nextColumn(2);
113
- }
114
-
115
- public void setInt(int v) throws IOException, SQLException
116
- {
117
- batch.setInt(index, v);
118
- nextColumn(4);
119
- }
120
-
121
- public void setLong(long v) throws IOException, SQLException
122
- {
123
- batch.setLong(index, v);
124
- nextColumn(8);
125
- }
126
-
127
- public void setFloat(float v) throws IOException, SQLException
128
- {
129
- batch.setFloat(index, v);
130
- nextColumn(4);
131
- }
132
-
133
- public void setDouble(double v) throws IOException, SQLException
134
- {
135
- batch.setDouble(index, v);
136
- nextColumn(8);
137
- }
138
-
139
- public void setBigDecimal(BigDecimal v) throws IOException, SQLException
140
- {
141
- // use estimated number of necessary bytes + 8 byte for the weight
142
- // assuming one place needs 4 bits. ceil(v.precision() / 2.0) + 8
143
- batch.setBigDecimal(index, v);
144
- nextColumn((v.precision() & ~2) / 2 + 8);
145
- }
146
-
147
- public void setString(String v) throws IOException, SQLException
148
- {
149
- batch.setString(index, v);
150
- // estimate all chracters use 2 bytes; almost enough for the worst case
151
- nextColumn(v.length() * 2 + 4);
152
- }
153
-
154
- public void setNString(String v) throws IOException, SQLException
155
- {
156
- batch.setNString(index, v);
157
- // estimate all chracters use 2 bytes; almost enough for the worst case
158
- nextColumn(v.length() * 2 + 4);
159
- }
160
-
161
- public void setBytes(byte[] v) throws IOException, SQLException
162
- {
163
- batch.setBytes(index, v);
164
- nextColumn(v.length + 4);
165
- }
166
-
167
- public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
168
- {
169
- // JavaDoc of java.sql.Time says:
170
- // >> To conform with the definition of SQL DATE, the millisecond values wrapped by a java.sql.Date instance must be 'normalized' by setting the hours, minutes, seconds, and milliseconds to zero in the particular time zone with which the instance is associated.
171
- cal.setTimeInMillis(v.getEpochSecond() * 1000);
172
- cal.set(Calendar.SECOND, 0);
173
- cal.set(Calendar.MINUTE, 0);
174
- cal.set(Calendar.HOUR_OF_DAY, 0);
175
- Date normalized = new Date(cal.getTimeInMillis());
176
- batch.setDate(index, normalized, cal);
177
- nextColumn(32);
178
- }
179
-
180
- public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
181
- {
182
- Time t = new Time(v.toEpochMilli());
183
- batch.setTime(index, t, cal);
184
- nextColumn(32);
185
- }
186
-
187
- public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
188
- {
189
- java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
190
- t.setNanos(v.getNano());
191
- batch.setTimestamp(index, t, cal);
192
- nextColumn(32);
193
- }
194
-
195
- private void nextColumn(int weight)
196
- {
197
- index++;
198
- batchWeight += weight + 4; // add weight as overhead of each columns
199
- }
200
- }
1
+ package org.embulk.output.jdbc;
2
+
3
+ import java.util.List;
4
+ import java.util.Calendar;
5
+ import java.io.IOException;
6
+ import java.math.BigDecimal;
7
+ import java.sql.PreparedStatement;
8
+ import java.sql.SQLException;
9
+ import java.sql.Date;
10
+ import java.sql.Time;
11
+ import com.google.common.base.Optional;
12
+ import org.slf4j.Logger;
13
+ import org.embulk.spi.time.Timestamp;
14
+ import org.embulk.spi.Exec;
15
+
16
+ public class StandardBatchInsert
17
+ implements BatchInsert
18
+ {
19
+ private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
20
+
21
+ private final JdbcOutputConnector connector;
22
+ private final Optional<List<String>> mergeKeys;
23
+
24
+ private JdbcOutputConnection connection;
25
+ private PreparedStatement batch;
26
+ private int index;
27
+ private int batchWeight;
28
+ private int batchRows;
29
+ private long totalRows;
30
+
31
+ public StandardBatchInsert(JdbcOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
32
+ {
33
+ this.connector = connector;
34
+ this.mergeKeys = mergeKeys;
35
+ }
36
+
37
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
38
+ {
39
+ this.connection = connector.connect(true);
40
+ this.index = 1; // PreparedStatement index begings from 1
41
+ this.batchRows = 0;
42
+ this.totalRows = 0;
43
+ this.batch = prepareStatement(loadTable, insertSchema);
44
+ batch.clearBatch();
45
+ }
46
+
47
+ protected PreparedStatement prepareStatement(String loadTable, JdbcSchema insertSchema) throws SQLException
48
+ {
49
+ return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeKeys);
50
+ }
51
+
52
+ public int getBatchWeight()
53
+ {
54
+ return batchWeight;
55
+ }
56
+
57
+ public void add() throws IOException, SQLException
58
+ {
59
+ batch.addBatch();
60
+ index = 1; // PreparedStatement index begins from 1
61
+ batchRows++;
62
+ batchWeight += 32; // add weight as overhead of each rows
63
+ }
64
+
65
+ public void close() throws IOException, SQLException
66
+ {
67
+ // caller should close the connection
68
+ }
69
+
70
+ public void flush() throws IOException, SQLException
71
+ {
72
+ logger.info(String.format("Loading %,d rows", batchRows));
73
+ long startTime = System.currentTimeMillis();
74
+ batch.executeBatch(); // here can't use returned value because MySQL Connector/J returns SUCCESS_NO_INFO as a batch result
75
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
76
+
77
+ totalRows += batchRows;
78
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
79
+ batch.clearBatch();
80
+ batchRows = 0;
81
+ batchWeight = 0;
82
+ }
83
+
84
+ public void finish() throws IOException, SQLException
85
+ {
86
+ if (getBatchWeight() != 0) {
87
+ flush();
88
+ }
89
+ }
90
+
91
+ public void setNull(int sqlType) throws IOException, SQLException
92
+ {
93
+ batch.setNull(index, sqlType);
94
+ nextColumn(0);
95
+ }
96
+
97
+ public void setBoolean(boolean v) throws IOException, SQLException
98
+ {
99
+ batch.setBoolean(index, v);
100
+ nextColumn(1);
101
+ }
102
+
103
+ public void setByte(byte v) throws IOException, SQLException
104
+ {
105
+ batch.setByte(index, v);
106
+ nextColumn(1);
107
+ }
108
+
109
+ public void setShort(short v) throws IOException, SQLException
110
+ {
111
+ batch.setShort(index, v);
112
+ nextColumn(2);
113
+ }
114
+
115
+ public void setInt(int v) throws IOException, SQLException
116
+ {
117
+ batch.setInt(index, v);
118
+ nextColumn(4);
119
+ }
120
+
121
+ public void setLong(long v) throws IOException, SQLException
122
+ {
123
+ batch.setLong(index, v);
124
+ nextColumn(8);
125
+ }
126
+
127
+ public void setFloat(float v) throws IOException, SQLException
128
+ {
129
+ batch.setFloat(index, v);
130
+ nextColumn(4);
131
+ }
132
+
133
+ public void setDouble(double v) throws IOException, SQLException
134
+ {
135
+ batch.setDouble(index, v);
136
+ nextColumn(8);
137
+ }
138
+
139
+ public void setBigDecimal(BigDecimal v) throws IOException, SQLException
140
+ {
141
+ // use estimated number of necessary bytes + 8 byte for the weight
142
+ // assuming one place needs 4 bits. ceil(v.precision() / 2.0) + 8
143
+ batch.setBigDecimal(index, v);
144
+ nextColumn((v.precision() & ~2) / 2 + 8);
145
+ }
146
+
147
+ public void setString(String v) throws IOException, SQLException
148
+ {
149
+ batch.setString(index, v);
150
+ // estimate all chracters use 2 bytes; almost enough for the worst case
151
+ nextColumn(v.length() * 2 + 4);
152
+ }
153
+
154
+ public void setNString(String v) throws IOException, SQLException
155
+ {
156
+ batch.setNString(index, v);
157
+ // estimate all chracters use 2 bytes; almost enough for the worst case
158
+ nextColumn(v.length() * 2 + 4);
159
+ }
160
+
161
+ public void setBytes(byte[] v) throws IOException, SQLException
162
+ {
163
+ batch.setBytes(index, v);
164
+ nextColumn(v.length + 4);
165
+ }
166
+
167
+ public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
168
+ {
169
+ // JavaDoc of java.sql.Time says:
170
+ // >> To conform with the definition of SQL DATE, the millisecond values wrapped by a java.sql.Date instance must be 'normalized' by setting the hours, minutes, seconds, and milliseconds to zero in the particular time zone with which the instance is associated.
171
+ cal.setTimeInMillis(v.getEpochSecond() * 1000);
172
+ cal.set(Calendar.SECOND, 0);
173
+ cal.set(Calendar.MINUTE, 0);
174
+ cal.set(Calendar.HOUR_OF_DAY, 0);
175
+ Date normalized = new Date(cal.getTimeInMillis());
176
+ batch.setDate(index, normalized, cal);
177
+ nextColumn(32);
178
+ }
179
+
180
+ public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
181
+ {
182
+ Time t = new Time(v.toEpochMilli());
183
+ batch.setTime(index, t, cal);
184
+ nextColumn(32);
185
+ }
186
+
187
+ public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
188
+ {
189
+ java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
190
+ t.setNanos(v.getNano());
191
+ batch.setTimestamp(index, t, cal);
192
+ nextColumn(32);
193
+ }
194
+
195
+ private void nextColumn(int weight)
196
+ {
197
+ index++;
198
+ batchWeight += weight + 4; // add weight as overhead of each columns
199
+ }
200
+ }