embulk-output-cdata 0.1.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b075cb5050a4ae67af18af9dcaea8b670168dc4b
4
- data.tar.gz: ad3b82b713cd6f1dc5ab8345b4e9d2ddf0871cf9
3
+ metadata.gz: 90fadee674529b7de2ad81d3ae7d4d541897f250
4
+ data.tar.gz: c4e6016c12c8d13e85050d737dfbe5fe85c26d67
5
5
  SHA512:
6
- metadata.gz: b45d0a77a371232cadbb3f9c5a0e60a612aff2b4360f2ebb8c5ea4536be17b4c43d15a5e8900d92035d015566ad5819d94d35d1c63d800ad63e1dc9b90fe0f2d
7
- data.tar.gz: ce31c62da4778242d8e0a6b5a8e0dd64fafc029712252a2922a2a4d1cf81f2f49291556bc89d783b69ab7d038cc49b2d0be43816521d3383775710d399bc397d
6
+ metadata.gz: 18e8c5c020dfb018aa6648df569dbae1e598e32b702baa58bd60a9ffdaf0f46bbbf9efc010d7fa20604090d9505fd78f1a740b14bd67f7fde45710ec649d4618
7
+ data.tar.gz: f090bf63e1c8a13f1abf4b3c172646e6d6da679ebf1d68fdcbea06f955a6cdd9d642e68c749eee71c8f6f369ce359131091159a165d498a21fbcb66665755682
data/README.md CHANGED
@@ -1,6 +1,6 @@
1
- # Cdata output plugin for Embulk
1
+ # CData output plugin for Embulk
2
2
 
3
- TODO: Write short description here and build.gradle file.
3
+ Support to CData JDBC Driver bulk upsert or update statements.
4
4
 
5
5
  ## Overview
6
6
 
@@ -11,17 +11,22 @@ TODO: Write short description here and build.gradle file.
11
11
 
12
12
  ## Configuration
13
13
 
14
- - **option1**: description (integer, required)
15
- - **option2**: description (string, default: `"myvalue"`)
16
- - **option3**: description (string, default: `null`)
14
+ - **driver_path**: description (string, required)
15
+ - **url**: description (string, `"jdbc:..."`)
16
+ - **mode**: description (string, `"insert_direct" | "update"`)
17
+ - **table**: description (string, required)
18
+ - **external_id_column**: description (string, required)
17
19
 
18
20
  ## Example
19
21
 
20
22
  ```yaml
21
23
  out:
22
24
  type: cdata
23
- option1: example1
24
- option2: example2
25
+ driver_path: lib/awesome.jar
26
+ url: jdbc:...
27
+ mode: merge_direct
28
+ table: TableName
29
+ external_id_column: ExternalIdColumn
25
30
  ```
26
31
 
27
32
 
data/build.gradle CHANGED
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.1.0"
16
+ version = "0.3.1"
17
17
 
18
18
  sourceCompatibility = 1.8
19
19
  targetCompatibility = 1.8
@@ -23,6 +23,9 @@ dependencies {
23
23
  provided "org.embulk:embulk-core:0.10.35"
24
24
  // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
25
25
  testCompile 'junit:junit:4.13.2'
26
+
27
+ implementation 'org.apache.logging.log4j:log4j-api:2.17.2'
28
+ implementation 'org.apache.logging.log4j:log4j-core:2.17.2'
26
29
  }
27
30
 
28
31
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -1,5 +1,8 @@
1
1
  package org.embulk.output.cdata;
2
2
 
3
+ import org.embulk.config.*;
4
+ import org.embulk.spi.*;
5
+
3
6
  import java.lang.reflect.InvocationTargetException;
4
7
  import java.lang.reflect.Method;
5
8
  import java.net.MalformedURLException;
@@ -10,108 +13,110 @@ import java.sql.Connection;
10
13
  import java.sql.DriverManager;
11
14
  import java.sql.SQLException;
12
15
  import java.util.List;
13
-
14
- import org.embulk.config.*;
15
- import org.embulk.spi.*;
16
+ import java.util.Objects;
16
17
 
17
18
  public class CDataOutputPlugin
18
- implements OutputPlugin
19
- {
19
+ implements OutputPlugin {
20
20
 
21
- private static Connection conn;
22
- public interface PluginTask
23
- extends Task
24
- {
25
- @Config("driver_path")
26
- public String getDriverPath();
21
+ private static Connection conn;
27
22
 
28
- @Config("url")
29
- public String getUrl();
23
+ public interface PluginTask
24
+ extends Task {
25
+ @Config("driver_path")
26
+ String getDriverPath();
30
27
 
31
- @Config("table")
32
- public String getTable();
33
- }
28
+ @Config("url")
29
+ String getUrl();
34
30
 
35
- @Override
36
- public ConfigDiff transaction(ConfigSource config,
37
- Schema schema, int taskCount,
38
- OutputPlugin.Control control)
39
- {
40
- PluginTask task = config.loadConfig(PluginTask.class);
41
-
42
- try {
43
- addDriverJarToClasspath(task.getDriverPath());
44
- Class.forName("cdata.jdbc.salesforce.SalesforceDriver");
45
- conn = DriverManager.getConnection(task.getUrl());
46
- } catch (ClassNotFoundException | SQLException e) {
47
- throw new RuntimeException(e);
48
- }
49
-
50
- // retryable (idempotent) output:
51
- // return resume(task.dump(), schema, taskCount, control);
52
-
53
- // non-retryable (non-idempotent) output:
54
- control.run(task.dump());
55
- return Exec.newConfigDiff();
56
- }
31
+ @Config("mode")
32
+ String getMode();
57
33
 
58
- @Override
59
- public ConfigDiff resume(TaskSource taskSource,
60
- Schema schema, int taskCount,
61
- OutputPlugin.Control control)
62
- {
63
- throw new UnsupportedOperationException("cdata output plugin does not support resuming");
64
- }
34
+ @Config("table")
35
+ String getTable();
36
+
37
+ @Config("external_id_column")
38
+ String getExternalIdColumn();
39
+ }
65
40
 
66
- @Override
67
- public void cleanup(TaskSource taskSource,
68
- Schema schema, int taskCount,
69
- List<TaskReport> successTaskReports)
70
- {
41
+ @Override
42
+ public ConfigDiff transaction(ConfigSource config,
43
+ Schema schema, int taskCount,
44
+ OutputPlugin.Control control) {
45
+ PluginTask task = config.loadConfig(PluginTask.class);
46
+
47
+ try {
48
+ addDriverJarToClasspath(task.getDriverPath());
49
+ Class.forName("cdata.jdbc.salesforce.SalesforceDriver");
50
+ conn = DriverManager.getConnection(task.getUrl());
51
+ } catch (ClassNotFoundException | SQLException e) {
52
+ throw new RuntimeException(e);
71
53
  }
72
54
 
73
- @Override
74
- public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
75
- {
76
- PluginTask task = taskSource.loadTask(PluginTask.class);
55
+ // retryable (idempotent) output:
56
+ // return resume(task.dump(), schema, taskCount, control);
57
+
58
+ // non-retryable (non-idempotent) output:
59
+ control.run(task.dump());
60
+ return Exec.newConfigDiff();
61
+ }
62
+
63
+ @Override
64
+ public ConfigDiff resume(TaskSource taskSource,
65
+ Schema schema, int taskCount,
66
+ OutputPlugin.Control control) {
67
+ throw new UnsupportedOperationException("cdata output plugin does not support resuming");
68
+ }
69
+
70
+ @Override
71
+ public void cleanup(TaskSource taskSource,
72
+ Schema schema, int taskCount,
73
+ List<TaskReport> successTaskReports) {
74
+ }
75
+
76
+ @Override
77
+ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
78
+ PluginTask task = taskSource.loadTask(PluginTask.class);
77
79
 
78
- PageReader reader = new PageReader(schema);
79
- return new CDataPageOutput(reader, conn, task);
80
+ PageReader reader = new PageReader(schema);
81
+ if (Objects.equals(task.getMode(), "insert_direct")) {
82
+ return new CDataPageOutputForUpsert(reader, conn, task);
83
+ } else {
84
+ return new CDataPageOutputForUpdate(reader, conn, task);
80
85
  }
86
+ }
81
87
 
82
- private void addDriverJarToClasspath(String glob)
83
- {
84
- // TODO match glob
85
- final ClassLoader loader = getClass().getClassLoader();
86
- if (!(loader instanceof URLClassLoader)) {
87
- throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
88
- }
89
- if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
90
- throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
91
- }
92
- Path path = Paths.get(glob);
93
- if (!path.toFile().exists()) {
94
- throw new ConfigException("The specified driver jar doesn't exist: " + glob);
95
- }
96
- final Method addPathMethod;
97
- try {
98
- addPathMethod = loader.getClass().getMethod("addPath", Path.class);
99
- } catch (final NoSuchMethodException ex) {
100
- throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
101
- }
102
- try {
103
- addPathMethod.invoke(loader, Paths.get(glob));
104
- } catch (final IllegalAccessException ex) {
105
- throw new RuntimeException(ex);
106
- } catch (final InvocationTargetException ex) {
107
- final Throwable targetException = ex.getTargetException();
108
- if (targetException instanceof MalformedURLException) {
109
- throw new IllegalArgumentException(targetException);
110
- } else if (targetException instanceof RuntimeException) {
111
- throw (RuntimeException) targetException;
112
- } else {
113
- throw new RuntimeException(targetException);
114
- }
115
- }
88
+ private void addDriverJarToClasspath(String glob) {
89
+ // TODO match glob
90
+ final ClassLoader loader = getClass().getClassLoader();
91
+ if (!(loader instanceof URLClassLoader)) {
92
+ throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
93
+ }
94
+ if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
95
+ throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
96
+ }
97
+ Path path = Paths.get(glob);
98
+ if (!path.toFile().exists()) {
99
+ throw new ConfigException("The specified driver jar doesn't exist: " + glob);
100
+ }
101
+ final Method addPathMethod;
102
+ try {
103
+ addPathMethod = loader.getClass().getMethod("addPath", Path.class);
104
+ } catch (final NoSuchMethodException ex) {
105
+ throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
106
+ }
107
+ try {
108
+ addPathMethod.invoke(loader, Paths.get(glob));
109
+ } catch (final IllegalAccessException ex) {
110
+ throw new RuntimeException(ex);
111
+ } catch (final InvocationTargetException ex) {
112
+ final Throwable targetException = ex.getTargetException();
113
+ if (targetException instanceof MalformedURLException) {
114
+ throw new IllegalArgumentException(targetException);
115
+ } else if (targetException instanceof RuntimeException) {
116
+ throw (RuntimeException) targetException;
117
+ } else {
118
+ throw new RuntimeException(targetException);
119
+ }
116
120
  }
121
+ }
117
122
  }
@@ -0,0 +1,200 @@
1
+ package org.embulk.output.cdata;
2
+
3
+ import org.embulk.config.TaskReport;
4
+ import org.embulk.spi.*;
5
+ import org.slf4j.Logger;
6
+ import org.slf4j.LoggerFactory;
7
+
8
+ import java.sql.*;
9
+ import java.util.ArrayList;
10
+ import java.util.HashMap;
11
+ import java.util.Objects;
12
+ import java.util.stream.Collectors;
13
+
14
+ public class CDataPageOutputForUpdate implements TransactionalPageOutput {
15
+
16
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForUpdate.class);
17
+
18
+ private final PageReader pageReader;
19
+ private final Connection conn;
20
+ private final CDataOutputPlugin.PluginTask task;
21
+ private PreparedStatement preparedStatement;
22
+ private String currentExternalIdColumn = "";
23
+
24
+ public CDataPageOutputForUpdate(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
25
+ this.pageReader = reader;
26
+ this.conn = conn;
27
+ this.task = task;
28
+ this.preparedStatement = null;
29
+ }
30
+
31
+ @Override
32
+ public void add(Page page) {
33
+ pageReader.setPage(page);
34
+
35
+ ArrayList<String> columnNamesWithId = new ArrayList<>();
36
+ columnNamesWithId.add("Id");
37
+ ArrayList<String> columnNames = pageReader.getSchema().getColumns().stream()
38
+ .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
39
+ columnNamesWithId.addAll(columnNames);
40
+
41
+ ArrayList<String> preparedValues = pageReader.getSchema().getColumns().stream()
42
+ .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
43
+ preparedValues.add("?"); // for Id
44
+
45
+ HashMap<String, String> idMap = new HashMap<>();
46
+ String selectStatement = "SELECT " + String.join(", ", columnNamesWithId) + " FROM " + task.getTable();
47
+ logger.info(selectStatement);
48
+ try {
49
+ Statement statement = conn.createStatement();
50
+ boolean ret = statement.execute(selectStatement);
51
+ if (ret) {
52
+ ResultSet rs = statement.getResultSet();
53
+ while (rs.next()) {
54
+ String id = "";
55
+ String externalIdColumn = "";
56
+ for (int i = 1; i <= rs.getMetaData().getColumnCount(); i++) {
57
+ if (Objects.equals(rs.getMetaData().getColumnName(i), "Id")) {
58
+ id = rs.getString(i);
59
+ } else if (Objects.equals(rs.getMetaData().getColumnName(i), task.getExternalIdColumn())) {
60
+ externalIdColumn = rs.getString(i);
61
+ }
62
+ }
63
+ idMap.put(externalIdColumn, id);
64
+ }
65
+ }
66
+ } catch (SQLException e) {
67
+ throw new RuntimeException(e);
68
+ }
69
+
70
+ String insertStatement = "INSERT INTO Temp#TEMP(" +
71
+ String.join(", ", columnNamesWithId) +
72
+ ") VALUES (" +
73
+ String.join(", ", preparedValues) + ")";
74
+ logger.info(insertStatement);
75
+
76
+ while (pageReader.nextRecord()) {
77
+ try {
78
+ this.preparedStatement = conn.prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS);
79
+
80
+ pageReader.getSchema().visitColumns(new ColumnVisitor() {
81
+ @Override
82
+ public void booleanColumn(Column column) {
83
+ try {
84
+ logger.info(column.getName() + ": " + pageReader.getBoolean(column));
85
+ preparedStatement.setBoolean(column.getIndex() + 2, pageReader.getBoolean(column));
86
+ } catch (SQLException e) {
87
+ throw new RuntimeException(e);
88
+ }
89
+ }
90
+
91
+ @Override
92
+ public void longColumn(Column column) {
93
+ try {
94
+ logger.info(column.getName() + ": " + pageReader.getLong(column));
95
+ preparedStatement.setLong(column.getIndex() + 2, pageReader.getLong(column));
96
+ } catch (SQLException e) {
97
+ throw new RuntimeException(e);
98
+ }
99
+ }
100
+
101
+ @Override
102
+ public void doubleColumn(Column column) {
103
+ try {
104
+ logger.info(column.getName() + ": " + pageReader.getDouble(column));
105
+ preparedStatement.setDouble(column.getIndex() + 2, pageReader.getDouble(column));
106
+ } catch (SQLException e) {
107
+ throw new RuntimeException(e);
108
+ }
109
+ }
110
+
111
+ @Override
112
+ public void stringColumn(Column column) {
113
+ try {
114
+ logger.info(column.getName() + ": " + pageReader.getString(column));
115
+ if (Objects.equals(column.getName(), task.getExternalIdColumn())) {
116
+ currentExternalIdColumn = pageReader.getString(column);
117
+ }
118
+ preparedStatement.setString(column.getIndex() + 2, pageReader.getString(column));
119
+ } catch (SQLException e) {
120
+ throw new RuntimeException(e);
121
+ }
122
+ }
123
+
124
+ @Override
125
+ public void timestampColumn(Column column) {
126
+ try {
127
+ logger.info(column.getName() + ": " + pageReader.getTimestampInstant(column));
128
+ preparedStatement.setTimestamp(column.getIndex() + 2, Timestamp.from(pageReader.getTimestampInstant(column)));
129
+ } catch (SQLException e) {
130
+ throw new RuntimeException(e);
131
+ }
132
+ }
133
+
134
+ @Override
135
+ public void jsonColumn(Column column) {
136
+ try {
137
+ logger.info(column.getName() + ": " + pageReader.getString(column));
138
+ preparedStatement.setString(column.getIndex() + 2, pageReader.getString(column));
139
+ } catch (SQLException e) {
140
+ throw new RuntimeException(e);
141
+ }
142
+ }
143
+ });
144
+ String id = idMap.get(currentExternalIdColumn);
145
+ if (id != null) {
146
+ preparedStatement.setString(1, id);
147
+ preparedStatement.executeUpdate();
148
+ logger.info("inserted to Temp#TEMP");
149
+ } else {
150
+ logger.info("skipped insert to Temp#TEMP, currentExternalIdColumn: " + currentExternalIdColumn);
151
+ }
152
+ } catch (SQLException e) {
153
+ throw new RuntimeException(e);
154
+ }
155
+ }
156
+ try {
157
+ String updateStatement = "UPDATE " + task.getTable() + " (" +
158
+ String.join(", ", columnNamesWithId) +
159
+ ") SELECT " +
160
+ String.join(", ", columnNamesWithId) +
161
+ " FROM Temp#TEMP";
162
+ logger.info(updateStatement);
163
+ conn.createStatement().executeUpdate(updateStatement, Statement.RETURN_GENERATED_KEYS);
164
+ } catch (SQLException e) {
165
+ throw new RuntimeException(e);
166
+ }
167
+ }
168
+
169
+ @Override
170
+ public void finish() {
171
+ try {
172
+ conn.close();
173
+ } catch (SQLException e) {
174
+ throw new RuntimeException(e);
175
+ }
176
+ }
177
+
178
+ @Override
179
+ public void close() {
180
+ try {
181
+ conn.close();
182
+ } catch (SQLException e) {
183
+ throw new RuntimeException(e);
184
+ }
185
+ }
186
+
187
+ @Override
188
+ public void abort() {
189
+ try {
190
+ conn.close();
191
+ } catch (SQLException e) {
192
+ throw new RuntimeException(e);
193
+ }
194
+ }
195
+
196
+ @Override
197
+ public TaskReport commit() {
198
+ return null;
199
+ }
200
+ }
@@ -2,17 +2,23 @@ package org.embulk.output.cdata;
2
2
 
3
3
  import org.embulk.config.TaskReport;
4
4
  import org.embulk.spi.*;
5
+ import org.slf4j.Logger;
6
+ import org.slf4j.LoggerFactory;
5
7
 
6
8
  import java.sql.*;
9
+ import java.util.ArrayList;
10
+ import java.util.stream.Collectors;
7
11
 
8
- public class CDataPageOutput implements TransactionalPageOutput {
12
+ public class CDataPageOutputForUpsert implements TransactionalPageOutput {
13
+
14
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForUpsert.class);
9
15
 
10
16
  private final PageReader pageReader;
11
17
  private final Connection conn;
12
18
  private final CDataOutputPlugin.PluginTask task;
13
19
  private PreparedStatement preparedStatement;
14
20
 
15
- public CDataPageOutput(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
21
+ public CDataPageOutputForUpsert(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
16
22
  this.pageReader = reader;
17
23
  this.conn = conn;
18
24
  this.task = task;
@@ -22,23 +28,30 @@ public class CDataPageOutput implements TransactionalPageOutput {
22
28
  @Override
23
29
  public void add(Page page) {
24
30
  pageReader.setPage(page);
25
- String[] columnNames = pageReader.getSchema().getColumns().stream().map(Column::getName).toArray(String[]::new);
26
- String[] preparedValues = pageReader.getSchema().getColumns().stream().map(it -> "?").toArray(String[]::new);
31
+ ArrayList<String> columnNames = pageReader.getSchema().getColumns().stream()
32
+ .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
33
+ columnNames.add("ExternalIdColumn");
27
34
 
28
- while (pageReader.nextRecord()) {
29
- String query = "INSERT INTO Temp#TEMP(" +
30
- String.join(", ", columnNames) +
31
- ") VALUES (" +
32
- String.join(", ", preparedValues) + ")";
35
+ ArrayList<String> preparedValues = pageReader.getSchema().getColumns().stream()
36
+ .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
37
+ preparedValues.add("?"); // for ExternalIdColumn
33
38
 
39
+ String insertStatement = "INSERT INTO Temp#TEMP(" +
40
+ String.join(", ", columnNames) +
41
+ ") VALUES (" +
42
+ String.join(", ", preparedValues) + ")";
43
+ logger.info(insertStatement);
44
+
45
+ while (pageReader.nextRecord()) {
34
46
  try {
35
- this.preparedStatement = conn.prepareStatement(query, Statement.RETURN_GENERATED_KEYS);
47
+ this.preparedStatement = conn.prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS);
36
48
 
37
49
  pageReader.getSchema().visitColumns(new ColumnVisitor() {
38
50
  @Override
39
51
  public void booleanColumn(Column column) {
40
52
  try {
41
- preparedStatement.setBoolean(column.getIndex(), pageReader.getBoolean(column));
53
+ logger.info(column.getName() + ": " + pageReader.getBoolean(column));
54
+ preparedStatement.setBoolean(column.getIndex() + 1, pageReader.getBoolean(column));
42
55
  } catch (SQLException e) {
43
56
  throw new RuntimeException(e);
44
57
  }
@@ -47,6 +60,7 @@ public class CDataPageOutput implements TransactionalPageOutput {
47
60
  @Override
48
61
  public void longColumn(Column column) {
49
62
  try {
63
+ logger.info(column.getName() + ": " + pageReader.getLong(column));
50
64
  preparedStatement.setLong(column.getIndex() + 1, pageReader.getLong(column));
51
65
  } catch (SQLException e) {
52
66
  throw new RuntimeException(e);
@@ -56,6 +70,7 @@ public class CDataPageOutput implements TransactionalPageOutput {
56
70
  @Override
57
71
  public void doubleColumn(Column column) {
58
72
  try {
73
+ logger.info(column.getName() + ": " + pageReader.getDouble(column));
59
74
  preparedStatement.setDouble(column.getIndex() + 1, pageReader.getDouble(column));
60
75
  } catch (SQLException e) {
61
76
  throw new RuntimeException(e);
@@ -65,6 +80,7 @@ public class CDataPageOutput implements TransactionalPageOutput {
65
80
  @Override
66
81
  public void stringColumn(Column column) {
67
82
  try {
83
+ logger.info(column.getName() + ": " + pageReader.getString(column));
68
84
  preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
69
85
  } catch (SQLException e) {
70
86
  throw new RuntimeException(e);
@@ -74,6 +90,7 @@ public class CDataPageOutput implements TransactionalPageOutput {
74
90
  @Override
75
91
  public void timestampColumn(Column column) {
76
92
  try {
93
+ logger.info(column.getName() + ": " + pageReader.getTimestampInstant(column));
77
94
  preparedStatement.setTimestamp(column.getIndex() + 1, Timestamp.from(pageReader.getTimestampInstant(column)));
78
95
  } catch (SQLException e) {
79
96
  throw new RuntimeException(e);
@@ -83,24 +100,28 @@ public class CDataPageOutput implements TransactionalPageOutput {
83
100
  @Override
84
101
  public void jsonColumn(Column column) {
85
102
  try {
103
+ logger.info(column.getName() + ": " + pageReader.getString(column));
86
104
  preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
87
105
  } catch (SQLException e) {
88
106
  throw new RuntimeException(e);
89
107
  }
90
108
  }
91
109
  });
110
+ preparedStatement.setString(preparedValues.size(), task.getExternalIdColumn());
92
111
  preparedStatement.executeUpdate();
112
+ logger.info("inserted to Temp#TEMP");
93
113
  } catch (SQLException e) {
94
114
  throw new RuntimeException(e);
95
115
  }
96
116
  }
97
117
  try {
98
- String query = "UPSERT INTO " + task.getTable() + " (" +
118
+ String upserStatement = "UPSERT INTO " + task.getTable() + " (" +
99
119
  String.join(", ", columnNames) +
100
120
  ") SELECT " +
101
121
  String.join(", ", columnNames) +
102
122
  " FROM Temp#TEMP";
103
- conn.createStatement().executeUpdate(query, Statement.RETURN_GENERATED_KEYS);
123
+ logger.info(upserStatement);
124
+ conn.createStatement().executeUpdate(upserStatement, Statement.RETURN_GENERATED_KEYS);
104
125
  } catch (SQLException e) {
105
126
  throw new RuntimeException(e);
106
127
  }
@@ -108,14 +129,29 @@ public class CDataPageOutput implements TransactionalPageOutput {
108
129
 
109
130
  @Override
110
131
  public void finish() {
132
+ try {
133
+ conn.close();
134
+ } catch (SQLException e) {
135
+ throw new RuntimeException(e);
136
+ }
111
137
  }
112
138
 
113
139
  @Override
114
140
  public void close() {
141
+ try {
142
+ conn.close();
143
+ } catch (SQLException e) {
144
+ throw new RuntimeException(e);
145
+ }
115
146
  }
116
147
 
117
148
  @Override
118
149
  public void abort() {
150
+ try {
151
+ conn.close();
152
+ } catch (SQLException e) {
153
+ throw new RuntimeException(e);
154
+ }
119
155
  }
120
156
 
121
157
  @Override
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-cdata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - sampo02
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-06-07 00:00:00.000000000 Z
11
+ date: 2022-06-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
- - classpath/embulk-output-cdata-0.1.0.jar
52
+ - classpath/embulk-output-cdata-0.3.1.jar
53
53
  - config/checkstyle/checkstyle.xml
54
54
  - config/checkstyle/default.xml
55
55
  - gradle/wrapper/gradle-wrapper.jar
@@ -58,7 +58,8 @@ files:
58
58
  - gradlew.bat
59
59
  - lib/embulk/output/cdata.rb
60
60
  - src/main/java/org/embulk/output/cdata/CDataOutputPlugin.java
61
- - src/main/java/org/embulk/output/cdata/CDataPageOutput.java
61
+ - src/main/java/org/embulk/output/cdata/CDataPageOutputForUpdate.java
62
+ - src/main/java/org/embulk/output/cdata/CDataPageOutputForUpsert.java
62
63
  - src/test/java/org/embulk/output/cdata/TestCDataOutputPlugin.java
63
64
  homepage:
64
65
  licenses: