embulk-output-cdata 0.3.4 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6f2031c04f4a622373cbb14b248c8c38650d0483
4
- data.tar.gz: 9fee98da33876dc3e8a22f89dfe23ba802cdc717
3
+ metadata.gz: 2a79b482391f04bdffabfda7cadb2bf8fffdcaea
4
+ data.tar.gz: 805ef6119099703dfa3591cfb1038f3bd6718657
5
5
  SHA512:
6
- metadata.gz: 7c4eabc9a60cc5760195a6a32cbf8c59bd4c5cecf1fcfa4daea36264d2f3e6c03e1bcba405fc68cca7bf562b7fb5e3ef9c5cc2cc4197732a8ff81ce278c10e47
7
- data.tar.gz: 2b2731fbc43c274b61a683ce13c3141633c435683b91c717c5823e37c08cc28fca70aeb0bb11ef2f93af9fffd5d22527db27281214bbf4d09c152f0fbe6a8398
6
+ metadata.gz: fa5485cffa86d2b25359d227a86dd7119ac06ecb1a4783acc21ee9db26c27411e279630f13fc67111c994945112956f8d85b0b83f6e62cdb3c63f994f79247e5
7
+ data.tar.gz: 06104206fdc4d697960b937dc92d0e26e8e23d520b9639ae5ad75358f2470626be8bc2f6237a8f98bf2098bd48f66747ebbaab2d2174daac6b9e4c130672114c
data/README.md CHANGED
@@ -12,10 +12,12 @@ Support to CData JDBC Driver bulk upsert or update statements.
12
12
  ## Configuration
13
13
 
14
14
  - **driver_path**: description (string, required)
15
+ - **class_name**: description (string, required)
15
16
  - **url**: description (string, `"jdbc:..."`)
16
- - **mode**: description (string, `"insert_direct" | "update"`)
17
+ - **mode**: description (string, `"upsert" | "update"`)
17
18
  - **table**: description (string, required)
18
19
  - **external_id_column**: description (string, required)
20
+ - **default_primary_key**: description (string, optional, default: "id")
19
21
 
20
22
  ## Example
21
23
 
@@ -23,8 +25,9 @@ Support to CData JDBC Driver bulk upsert or update statements.
23
25
  out:
24
26
  type: cdata
25
27
  driver_path: lib/awesome.jar
28
+ class_name: yourAwesomeClassName
26
29
  url: jdbc:...
27
- mode: merge_direct
30
+ mode: upsert
28
31
  table: TableName
29
32
  external_id_column: ExternalIdColumn
30
33
  ```
data/build.gradle CHANGED
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.3.4"
16
+ version = "0.4.1"
17
17
 
18
18
  sourceCompatibility = 1.8
19
19
  targetCompatibility = 1.8
@@ -25,6 +25,9 @@ public class CDataOutputPlugin
25
25
  @Config("driver_path")
26
26
  String getDriverPath();
27
27
 
28
+ @Config("class_name")
29
+ String getDriverName();
30
+
28
31
  @Config("url")
29
32
  String getUrl();
30
33
 
@@ -36,17 +39,25 @@ public class CDataOutputPlugin
36
39
 
37
40
  @Config("external_id_column")
38
41
  String getExternalIdColumn();
42
+
43
+ @Config("default_primary_key")
44
+ @ConfigDefault("\"id\"")
45
+ String getDefaultPrimaryKey();
46
+
47
+ @Config("remove_csv_file")
48
+ @ConfigDefault("false")
49
+ Boolean getRemoveCsvFile();
39
50
  }
40
51
 
41
52
  @Override
42
53
  public ConfigDiff transaction(ConfigSource config,
43
54
  Schema schema, int taskCount,
44
- OutputPlugin.Control control) {
55
+ Control control) {
45
56
  PluginTask task = config.loadConfig(PluginTask.class);
46
57
 
47
58
  try {
48
59
  addDriverJarToClasspath(task.getDriverPath());
49
- Class.forName("cdata.jdbc.salesforce.SalesforceDriver");
60
+ Class.forName(task.getDriverName());
50
61
  conn = DriverManager.getConnection(task.getUrl());
51
62
  } catch (ClassNotFoundException | SQLException e) {
52
63
  throw new RuntimeException(e);
@@ -63,7 +74,7 @@ public class CDataOutputPlugin
63
74
  @Override
64
75
  public ConfigDiff resume(TaskSource taskSource,
65
76
  Schema schema, int taskCount,
66
- OutputPlugin.Control control) {
77
+ Control control) {
67
78
  throw new UnsupportedOperationException("cdata output plugin does not support resuming");
68
79
  }
69
80
 
@@ -78,8 +89,15 @@ public class CDataOutputPlugin
78
89
  PluginTask task = taskSource.loadTask(PluginTask.class);
79
90
 
80
91
  PageReader reader = new PageReader(schema);
81
- if (Objects.equals(task.getMode(), "insert_direct")) {
82
- return new CDataPageOutputForUpsert(reader, conn, task);
92
+ if (Objects.equals(task.getMode(), "insert")) {
93
+ return new CDataPageOutputForInsert(reader, conn, task);
94
+ }
95
+ else if (Objects.equals(task.getMode(), "upsert")) {
96
+ if (Objects.equals(task.getDriverName(), "cdata.jdbc.salesforce.SalesforceDriver")) {
97
+ return new CDataPageOutputForUpsert(reader, conn, task);
98
+ } else {
99
+ return new CDataPageOutputForManualUpsert(reader, conn, task);
100
+ }
83
101
  } else {
84
102
  return new CDataPageOutputForUpdate(reader, conn, task);
85
103
  }
@@ -0,0 +1,177 @@
1
+ package org.embulk.output.cdata;
2
+
3
+ import org.embulk.config.TaskReport;
4
+ import org.embulk.output.cdata.procedures.DeleteFile;
5
+ import org.embulk.spi.*;
6
+ import org.slf4j.Logger;
7
+ import org.slf4j.LoggerFactory;
8
+
9
+ import java.sql.*;
10
+ import java.util.ArrayList;
11
+ import java.util.stream.Collectors;
12
+
13
+ public class CDataPageOutputForInsert implements TransactionalPageOutput {
14
+
15
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForInsert.class);
16
+
17
+ private final PageReader pageReader;
18
+ private final Connection conn;
19
+ private final CDataOutputPlugin.PluginTask task;
20
+ private PreparedStatement preparedStatement;
21
+
22
+ public CDataPageOutputForInsert(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
23
+ this.pageReader = reader;
24
+ this.conn = conn;
25
+ this.task = task;
26
+ this.preparedStatement = null;
27
+ }
28
+
29
+ @Override
30
+ public void add(Page page) {
31
+ pageReader.setPage(page);
32
+
33
+ if (task.getRemoveCsvFile()) {
34
+ try {
35
+ DeleteFile.execute(conn, task.getTable());
36
+ } catch (SQLException e) {
37
+ throw new RuntimeException(e);
38
+ }
39
+ }
40
+
41
+ ArrayList<String> columnNames = pageReader.getSchema().getColumns().stream()
42
+ .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
43
+
44
+ ArrayList<String> preparedValues = pageReader.getSchema().getColumns().stream()
45
+ .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
46
+
47
+ String insertStatement = "INSERT INTO " + task.getTable() + "(" +
48
+ String.join(", ", columnNames) +
49
+ ") VALUES (" +
50
+ String.join(", ", preparedValues) + ")";
51
+ logger.info(insertStatement);
52
+
53
+ while (pageReader.nextRecord()) {
54
+ try {
55
+ this.preparedStatement = conn.prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS);
56
+ pageReader.getSchema().visitColumns(createColumnVisitor(preparedStatement));
57
+ preparedStatement.executeUpdate();
58
+ } catch (SQLException e) {
59
+ throw new RuntimeException(e);
60
+ }
61
+ }
62
+ }
63
+
64
+ @Override
65
+ public void finish() {
66
+ try {
67
+ conn.close();
68
+ } catch (SQLException e) {
69
+ throw new RuntimeException(e);
70
+ }
71
+ }
72
+
73
+ @Override
74
+ public void close() {
75
+ try {
76
+ conn.close();
77
+ } catch (SQLException e) {
78
+ throw new RuntimeException(e);
79
+ }
80
+ }
81
+
82
+ @Override
83
+ public void abort() {
84
+ try {
85
+ conn.close();
86
+ } catch (SQLException e) {
87
+ throw new RuntimeException(e);
88
+ }
89
+ }
90
+
91
+ @Override
92
+ public TaskReport commit() {
93
+ return null;
94
+ }
95
+
96
+ private ColumnVisitor createColumnVisitor(PreparedStatement preparedStatement) {
97
+ return new ColumnVisitor() {
98
+ @Override
99
+ public void booleanColumn(Column column) {
100
+ try {
101
+ if (pageReader.isNull(column)) {
102
+ preparedStatement.setObject(column.getIndex() + 1, null);
103
+ } else {
104
+ preparedStatement.setBoolean(column.getIndex() + 1, pageReader.getBoolean(column));
105
+ }
106
+ } catch (SQLException e) {
107
+ throw new RuntimeException(e);
108
+ }
109
+ }
110
+
111
+ @Override
112
+ public void longColumn(Column column) {
113
+ try {
114
+ if (pageReader.isNull(column)) {
115
+ preparedStatement.setObject(column.getIndex() + 1, null);
116
+ } else {
117
+ preparedStatement.setLong(column.getIndex() + 1, pageReader.getLong(column));
118
+ }
119
+ } catch (SQLException e) {
120
+ throw new RuntimeException(e);
121
+ }
122
+ }
123
+
124
+ @Override
125
+ public void doubleColumn(Column column) {
126
+ try {
127
+ if (pageReader.isNull(column)) {
128
+ preparedStatement.setObject(column.getIndex() + 1, null);
129
+ } else {
130
+ preparedStatement.setDouble(column.getIndex() + 1, pageReader.getDouble(column));
131
+ }
132
+ } catch (SQLException e) {
133
+ throw new RuntimeException(e);
134
+ }
135
+ }
136
+
137
+ @Override
138
+ public void stringColumn(Column column) {
139
+ try {
140
+ if (pageReader.isNull(column)) {
141
+ preparedStatement.setObject(column.getIndex() + 1, null);
142
+ } else {
143
+ preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
144
+ }
145
+ } catch (SQLException e) {
146
+ throw new RuntimeException(e);
147
+ }
148
+ }
149
+
150
+ @Override
151
+ public void timestampColumn(Column column) {
152
+ try {
153
+ if (pageReader.isNull(column)) {
154
+ preparedStatement.setObject(column.getIndex() + 1, null);
155
+ } else {
156
+ preparedStatement.setTimestamp(column.getIndex() + 1, Timestamp.from(pageReader.getTimestampInstant(column)));
157
+ }
158
+ } catch (SQLException e) {
159
+ throw new RuntimeException(e);
160
+ }
161
+ }
162
+
163
+ @Override
164
+ public void jsonColumn(Column column) {
165
+ try {
166
+ if (pageReader.isNull(column)) {
167
+ preparedStatement.setObject(column.getIndex() + 1, null);
168
+ } else {
169
+ preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
170
+ }
171
+ } catch (SQLException e) {
172
+ throw new RuntimeException(e);
173
+ }
174
+ }
175
+ };
176
+ }
177
+ }
@@ -0,0 +1,148 @@
1
+ package org.embulk.output.cdata;
2
+
3
+ import org.embulk.spi.Column;
4
+ import org.embulk.spi.PageReader;
5
+ import org.slf4j.Logger;
6
+ import org.slf4j.LoggerFactory;
7
+
8
+ import java.sql.*;
9
+ import java.util.*;
10
+ import java.util.stream.Collectors;
11
+
12
+ public class CDataPageOutputForManualUpsert extends CDataPageOutputForUpsertBase {
13
+ private final String INSERT_TEMP_TABLE = "InsertTemp#TEMP";
14
+ private final String UPDATE_TEMP_TABLE = "UpdateTemp#TEMP";
15
+
16
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForManualUpsert.class);
17
+
18
+ public CDataPageOutputForManualUpsert(PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
19
+ super(reader, conn, task);
20
+ }
21
+
22
+ protected List<String> createColumns() {
23
+ return getPageReader().getSchema().getColumns().stream()
24
+ .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
25
+ }
26
+
27
+ protected List<String> createPlaceHolders() {
28
+ return getPageReader().getSchema().getColumns().stream()
29
+ .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
30
+ }
31
+
32
+ protected void executeInsert(List<String> columnNames, List<String> preparedValues) {
33
+ String externalIdColumn = getTask().getExternalIdColumn();
34
+ String primaryKeyColumn = getTask().getDefaultPrimaryKey();
35
+
36
+ try {
37
+ // split insert to InsertTemp#TEMP and UpdateTemp#TEMP
38
+ List<String> extractColumnNames = new ArrayList<>(Arrays.asList(externalIdColumn, primaryKeyColumn));
39
+ if (Objects.equals(externalIdColumn, primaryKeyColumn)) {
40
+ extractColumnNames = new ArrayList<>(Collections.singletonList(externalIdColumn));
41
+ }
42
+
43
+ ResultSet resultSet = selectRecordAll(getTask().getTable(), extractColumnNames);
44
+ Map<String, String> externalIdValueAndPrimaryKeyValueMap = toIds(
45
+ externalIdColumn,
46
+ primaryKeyColumn,
47
+ resultSet);
48
+
49
+ while (getPageReader().nextRecord()) {
50
+
51
+ int externalIdColumnIndex = columnNames.indexOf(externalIdColumn);
52
+ if (externalIdColumnIndex == -1) {
53
+ throw new RuntimeException("ExternalIdColumn is not found.");
54
+ }
55
+
56
+ String externalIdValue = getPageReader().getString(externalIdColumnIndex);
57
+ if (externalIdValueAndPrimaryKeyValueMap.containsKey(externalIdValue)) {
58
+ String PrimaryKeyColumnValue = externalIdValueAndPrimaryKeyValueMap.get(externalIdValue);
59
+
60
+ List<String> updateColumnNames = new ArrayList<>(columnNames);
61
+ List<String> updatePreparedValues = new ArrayList<>(preparedValues);
62
+
63
+ boolean mustBeAddPrimaryKey = !Objects.equals(getTask().getDefaultPrimaryKey(), getTask().getExternalIdColumn());
64
+ if (mustBeAddPrimaryKey) {
65
+ updateColumnNames.add(getTask().getDefaultPrimaryKey());
66
+ updatePreparedValues.add(PrimaryKeyColumnValue);
67
+ }
68
+
69
+ String updateStatement = createInsertQuery(UPDATE_TEMP_TABLE, updateColumnNames, updatePreparedValues);
70
+
71
+ // already record, insert to UpdateTemp#TEMP
72
+ try (PreparedStatement updatePreparedStatement = getConnection().prepareStatement(updateStatement, Statement.RETURN_GENERATED_KEYS)) {
73
+ getPageReader().getSchema().visitColumns(createColumnVisitor(updatePreparedStatement));
74
+ updatePreparedStatement.executeUpdate();
75
+ }
76
+
77
+ logger.info("inserted to " + UPDATE_TEMP_TABLE);
78
+
79
+ } else {
80
+ // new record, insert to InsertTemp#TEMP
81
+ String insertStatement = createInsertQuery(INSERT_TEMP_TABLE, columnNames, preparedValues);
82
+ try (PreparedStatement insertPreparedStatement = getConnection().prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS)) {
83
+ getPageReader().getSchema().visitColumns(createColumnVisitor(insertPreparedStatement));
84
+ insertPreparedStatement.executeUpdate();
85
+ } catch (SQLException e) {
86
+ throw new RuntimeException(e);
87
+ }
88
+
89
+ logger.info("inserted to " + INSERT_TEMP_TABLE);
90
+ }
91
+ }
92
+ } catch (SQLException e) {
93
+ throw new RuntimeException(e);
94
+ }
95
+ }
96
+
97
+ protected String createInsertIntoSelectQuery(String tableName, List<String> columnNames) {
98
+ return "INSERT INTO " + tableName + " (" +
99
+ String.join(", ", columnNames) +
100
+ ") SELECT " +
101
+ String.join(", ", columnNames) +
102
+ " FROM " + INSERT_TEMP_TABLE;
103
+ }
104
+
105
+ public String createUpdateIntoSelectQuery(String tableName, List<String> columnNames) {
106
+ return "UPDATE " + tableName + " (" +
107
+ String.join(", ", columnNames) +
108
+ ") SELECT " +
109
+ String.join(", ", columnNames) +
110
+ " FROM " + UPDATE_TEMP_TABLE;
111
+ }
112
+
113
+ protected String executeUpsert(String tableName, List<String> columnNames) throws SQLException {
114
+
115
+ String insertIntoSelectQuery = createInsertIntoSelectQuery(tableName, columnNames);
116
+ String updateIntoSelectQuery = createUpdateIntoSelectQuery(tableName, columnNames);
117
+
118
+ getConnection()
119
+ .createStatement()
120
+ .executeUpdate(insertIntoSelectQuery, Statement.RETURN_GENERATED_KEYS);
121
+
122
+ getConnection()
123
+ .createStatement()
124
+ .executeUpdate(updateIntoSelectQuery, Statement.RETURN_GENERATED_KEYS);
125
+
126
+ return insertIntoSelectQuery + " , and, " + updateIntoSelectQuery;
127
+ }
128
+
129
+ protected ResultSet selectRecordAll(String tableName, List<String> selectColumns) {
130
+ try {
131
+ Statement selectStatement = getConnection().createStatement();
132
+ return selectStatement.executeQuery("SELECT " + String.join(", ", selectColumns) + " FROM " + tableName);
133
+
134
+ } catch (SQLException e) {
135
+ throw new RuntimeException(e);
136
+ }
137
+ }
138
+
139
+ protected Map<String, String> toIds(String externalIdColumnName, String defaultPrimaryKey, ResultSet resultSet) throws SQLException {
140
+ Map<String, String> externalIdMap = new HashMap<>();
141
+ while (resultSet.next()) {
142
+ externalIdMap.put(resultSet.getString(externalIdColumnName), resultSet.getString(defaultPrimaryKey));
143
+ }
144
+
145
+ return externalIdMap;
146
+ }
147
+
148
+ }
@@ -1,161 +1,72 @@
1
1
  package org.embulk.output.cdata;
2
2
 
3
- import org.embulk.config.TaskReport;
4
- import org.embulk.spi.*;
3
+ import org.embulk.spi.Column;
4
+ import org.embulk.spi.PageReader;
5
5
  import org.slf4j.Logger;
6
6
  import org.slf4j.LoggerFactory;
7
7
 
8
8
  import java.sql.*;
9
9
  import java.util.ArrayList;
10
+ import java.util.List;
10
11
  import java.util.stream.Collectors;
11
12
 
12
- public class CDataPageOutputForUpsert implements TransactionalPageOutput {
13
+ public class CDataPageOutputForUpsert extends CDataPageOutputForUpsertBase {
13
14
 
14
- private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForUpsert.class);
15
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForUpsert.class);
15
16
 
16
- private final PageReader pageReader;
17
- private final Connection conn;
18
- private final CDataOutputPlugin.PluginTask task;
19
- private PreparedStatement preparedStatement;
17
+ public CDataPageOutputForUpsert(PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
18
+ super(reader, conn, task);
19
+ }
20
20
 
21
- public CDataPageOutputForUpsert(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
22
- this.pageReader = reader;
23
- this.conn = conn;
24
- this.task = task;
25
- this.preparedStatement = null;
26
- }
21
+ protected List<String> createColumns() {
22
+ List<String> columnNames = getPageReader().getSchema().getColumns().stream()
23
+ .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
27
24
 
28
- @Override
29
- public void add(Page page) {
30
- pageReader.setPage(page);
31
- ArrayList<String> columnNames = pageReader.getSchema().getColumns().stream()
32
- .map(Column::getName).collect(Collectors.toCollection(ArrayList::new));
33
- columnNames.add("ExternalIdColumn");
25
+ columnNames.add("ExternalIdColumn");
26
+ return columnNames;
27
+ }
34
28
 
35
- ArrayList<String> preparedValues = pageReader.getSchema().getColumns().stream()
36
- .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
37
- preparedValues.add("?"); // for ExternalIdColumn
29
+ protected List<String> createPlaceHolders() {
30
+ List<String> preparedValues = getPageReader().getSchema().getColumns().stream()
31
+ .map(it -> "?").collect(Collectors.toCollection(ArrayList::new));
32
+ preparedValues.add("?");
33
+ return preparedValues;
34
+ }
38
35
 
39
- String insertStatement = "INSERT INTO Temp#TEMP(" +
40
- String.join(", ", columnNames) +
41
- ") VALUES (" +
42
- String.join(", ", preparedValues) + ")";
43
- logger.info(insertStatement);
36
+ protected String createUpsertQuery(String tableName, List<String> columnNames) {
37
+ return "UPSERT INTO " + tableName + " (" +
38
+ String.join(", ", columnNames) +
39
+ ") SELECT " +
40
+ String.join(", ", columnNames) +
41
+ " FROM Temp#TEMP";
42
+ }
44
43
 
45
- while (pageReader.nextRecord()) {
46
- try {
47
- this.preparedStatement = conn.prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS);
44
+ protected void executeInsert(List<String> columnNames, List<String> preparedValues) throws SQLException {
45
+ String insertStatement = createInsertQuery("Temp#TEMP", columnNames, preparedValues);
46
+ logger.info(insertStatement);
48
47
 
49
- pageReader.getSchema().visitColumns(new ColumnVisitor() {
50
- @Override
51
- public void booleanColumn(Column column) {
52
- try {
53
- logger.info(column.getName() + ": " + pageReader.getBoolean(column));
54
- preparedStatement.setBoolean(column.getIndex() + 1, pageReader.getBoolean(column));
55
- } catch (SQLException e) {
56
- throw new RuntimeException(e);
57
- }
58
- }
48
+ PageReader pageReader = getPageReader();
49
+ Connection conn = getConnection();
50
+ CDataOutputPlugin.PluginTask task = getTask();
59
51
 
60
- @Override
61
- public void longColumn(Column column) {
52
+ while (pageReader.nextRecord()) {
62
53
  try {
63
- logger.info(column.getName() + ": " + pageReader.getLong(column));
64
- preparedStatement.setLong(column.getIndex() + 1, pageReader.getLong(column));
65
- } catch (SQLException e) {
66
- throw new RuntimeException(e);
67
- }
68
- }
54
+ PreparedStatement preparedStatement = conn.prepareStatement(insertStatement, Statement.RETURN_GENERATED_KEYS);
69
55
 
70
- @Override
71
- public void doubleColumn(Column column) {
72
- try {
73
- logger.info(column.getName() + ": " + pageReader.getDouble(column));
74
- preparedStatement.setDouble(column.getIndex() + 1, pageReader.getDouble(column));
75
- } catch (SQLException e) {
76
- throw new RuntimeException(e);
77
- }
78
- }
56
+ pageReader.getSchema().visitColumns(createColumnVisitor(preparedStatement));
57
+ preparedStatement.executeUpdate();
58
+ preparedStatement.setString(preparedValues.size(), task.getExternalIdColumn());
79
59
 
80
- @Override
81
- public void stringColumn(Column column) {
82
- try {
83
- logger.info(column.getName() + ": " + pageReader.getString(column));
84
- preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
60
+ logger.info("inserted to Temp#TEMP");
85
61
  } catch (SQLException e) {
86
- throw new RuntimeException(e);
62
+ throw new RuntimeException(e);
87
63
  }
88
- }
89
-
90
- @Override
91
- public void timestampColumn(Column column) {
92
- try {
93
- logger.info(column.getName() + ": " + pageReader.getTimestampInstant(column));
94
- preparedStatement.setTimestamp(column.getIndex() + 1, Timestamp.from(pageReader.getTimestampInstant(column)));
95
- } catch (SQLException e) {
96
- throw new RuntimeException(e);
97
- }
98
- }
99
-
100
- @Override
101
- public void jsonColumn(Column column) {
102
- try {
103
- logger.info(column.getName() + ": " + pageReader.getString(column));
104
- preparedStatement.setString(column.getIndex() + 1, pageReader.getString(column));
105
- } catch (SQLException e) {
106
- throw new RuntimeException(e);
107
- }
108
- }
109
- });
110
- preparedStatement.setString(preparedValues.size(), task.getExternalIdColumn());
111
- preparedStatement.executeUpdate();
112
- logger.info("inserted to Temp#TEMP");
113
- } catch (SQLException e) {
114
- throw new RuntimeException(e);
115
- }
64
+ }
116
65
  }
117
- try {
118
- String upserStatement = "UPSERT INTO " + task.getTable() + " (" +
119
- String.join(", ", columnNames) +
120
- ") SELECT " +
121
- String.join(", ", columnNames) +
122
- " FROM Temp#TEMP";
123
- logger.info(upserStatement);
124
- conn.createStatement().executeUpdate(upserStatement, Statement.RETURN_GENERATED_KEYS);
125
- } catch (SQLException e) {
126
- throw new RuntimeException(e);
127
- }
128
- }
129
-
130
- @Override
131
- public void finish() {
132
- try {
133
- conn.close();
134
- } catch (SQLException e) {
135
- throw new RuntimeException(e);
136
- }
137
- }
138
66
 
139
- @Override
140
- public void close() {
141
- try {
142
- conn.close();
143
- } catch (SQLException e) {
144
- throw new RuntimeException(e);
67
+ protected String executeUpsert(String tableName, List<String> columnNames) throws SQLException {
68
+ String upsertStatement = createUpsertQuery(tableName, columnNames);
69
+ getConnection().createStatement().executeUpdate(upsertStatement, Statement.RETURN_GENERATED_KEYS);
70
+ return upsertStatement;
145
71
  }
146
- }
147
-
148
- @Override
149
- public void abort() {
150
- try {
151
- conn.close();
152
- } catch (SQLException e) {
153
- throw new RuntimeException(e);
154
- }
155
- }
156
-
157
- @Override
158
- public TaskReport commit() {
159
- return null;
160
- }
161
72
  }
@@ -0,0 +1,196 @@
1
+ package org.embulk.output.cdata;
2
+
3
+ import org.embulk.config.TaskReport;
4
+ import org.embulk.spi.*;
5
+ import org.slf4j.Logger;
6
+ import org.slf4j.LoggerFactory;
7
+
8
+ import java.sql.*;
9
+ import java.util.List;
10
+
11
+ public class CDataPageOutputForUpsertBase implements TransactionalPageOutput {
12
+
13
+ private static final Logger logger = LoggerFactory.getLogger(CDataPageOutputForUpsertBase.class);
14
+
15
+ private final PageReader pageReader;
16
+ private final Connection conn;
17
+ private final CDataOutputPlugin.PluginTask task;
18
+
19
+ public CDataPageOutputForUpsertBase(final PageReader reader, Connection conn, CDataOutputPlugin.PluginTask task) {
20
+ this.pageReader = reader;
21
+ this.conn = conn;
22
+ this.task = task;
23
+ }
24
+
25
+ @Override
26
+ public void add(Page page) {
27
+ pageReader.setPage(page);
28
+ List<String> columnNames = createColumns();
29
+ List<String> preparedValues = createPlaceHolders(); // for ExternalIdColumn
30
+
31
+ try {
32
+ executeInsert(columnNames, preparedValues);
33
+ } catch (SQLException e) {
34
+ throw new RuntimeException(e);
35
+ }
36
+
37
+ try {
38
+ String upsertStatement = executeUpsert(task.getTable(), columnNames);
39
+ logger.info(upsertStatement);
40
+ } catch (SQLException e) {
41
+ throw new RuntimeException(e);
42
+ }
43
+ }
44
+
45
+ @Override
46
+ public void finish() {
47
+ try {
48
+ conn.close();
49
+ } catch (SQLException e) {
50
+ throw new RuntimeException(e);
51
+ }
52
+ }
53
+
54
+ @Override
55
+ public void close() {
56
+ try {
57
+ conn.close();
58
+ } catch (SQLException e) {
59
+ throw new RuntimeException(e);
60
+ }
61
+ }
62
+
63
+ @Override
64
+ public void abort() {
65
+ try {
66
+ conn.close();
67
+ } catch (SQLException e) {
68
+ throw new RuntimeException(e);
69
+ }
70
+ }
71
+
72
+ @Override
73
+ public TaskReport commit() {
74
+ return null;
75
+ }
76
+
77
+ protected PageReader getPageReader() {
78
+ return this.pageReader;
79
+ }
80
+
81
+ protected Connection getConnection() {
82
+ return this.conn;
83
+ }
84
+
85
+ protected CDataOutputPlugin.PluginTask getTask() {
86
+ return this.task;
87
+ }
88
+
89
+ protected List<String> createColumns() {
90
+ throw new UnsupportedOperationException("createColumns is not implemented");
91
+ }
92
+
93
+ protected List<String> createPlaceHolders() {
94
+ throw new UnsupportedOperationException("createPlaceHolders is not implemented");
95
+ }
96
+
97
+ /**
98
+ * Create placeholder insert query
99
+ * @param tableName
100
+ * @param columnNames
101
+ * @param preparedValues
102
+ * @return
103
+ */
104
+ protected String createInsertQuery(String tableName, List<String> columnNames, List<String> preparedValues) {
105
+ return "INSERT INTO " + tableName + "(" +
106
+ String.join(", ", columnNames) +
107
+ ") VALUES (" +
108
+ String.join(", ", preparedValues) + ")";
109
+ }
110
+
111
+ /**
112
+ * insert into Temp table
113
+ * @param columnNames
114
+ * @param preparedValues
115
+ * @throws SQLException
116
+ */
117
+ protected void executeInsert(List<String> columnNames, List<String> preparedValues) throws SQLException {
118
+ throw new UnsupportedOperationException("executeInsert is not implemented");
119
+ }
120
+
121
+ /**
122
+ * execute upsert query
123
+ * when not upsert query un suppoerted, must be use "INSERT INTO SELECT" and "UPDATE SELECT" in this method
124
+ * @param tableName
125
+ * @param columnNames
126
+ * @return query string, use for logging
127
+ * @throws SQLException
128
+ */
129
+ protected String executeUpsert(String tableName, List<String> columnNames) throws SQLException {
130
+ throw new RuntimeException("Not implemented");
131
+ }
132
+
133
+ protected ColumnVisitor createColumnVisitor(PreparedStatement preparedStatement) {
134
+ return new ColumnVisitor() {
135
+ @Override
136
+ public void booleanColumn(Column column) {
137
+ try {
138
+ logger.info(column.getName() + ": " + getPageReader().getBoolean(column));
139
+ preparedStatement.setBoolean(column.getIndex() + 1, getPageReader().getBoolean(column));
140
+ } catch (SQLException e) {
141
+ throw new RuntimeException(e);
142
+ }
143
+ }
144
+
145
+ @Override
146
+ public void longColumn(Column column) {
147
+ try {
148
+ logger.info(column.getName() + ": " + getPageReader().getLong(column));
149
+ preparedStatement.setLong(column.getIndex() + 1, getPageReader().getLong(column));
150
+ } catch (SQLException e) {
151
+ throw new RuntimeException(e);
152
+ }
153
+ }
154
+
155
+ @Override
156
+ public void doubleColumn(Column column) {
157
+ try {
158
+ logger.info(column.getName() + ": " + getPageReader().getDouble(column));
159
+ preparedStatement.setDouble(column.getIndex() + 1, getPageReader().getDouble(column));
160
+ } catch (SQLException e) {
161
+ throw new RuntimeException(e);
162
+ }
163
+ }
164
+
165
+ @Override
166
+ public void stringColumn(Column column) {
167
+ try {
168
+ logger.info(column.getName() + ": " + getPageReader().getString(column));
169
+ preparedStatement.setString(column.getIndex() + 1, getPageReader().getString(column));
170
+ } catch (SQLException e) {
171
+ throw new RuntimeException(e);
172
+ }
173
+ }
174
+
175
+ @Override
176
+ public void timestampColumn(Column column) {
177
+ try {
178
+ logger.info(column.getName() + ": " + getPageReader().getTimestampInstant(column));
179
+ preparedStatement.setTimestamp(column.getIndex() + 1, Timestamp.from(getPageReader().getTimestampInstant(column)));
180
+ } catch (SQLException e) {
181
+ throw new RuntimeException(e);
182
+ }
183
+ }
184
+
185
+ @Override
186
+ public void jsonColumn(Column column) {
187
+ try {
188
+ logger.info(column.getName() + ": " + getPageReader().getString(column));
189
+ preparedStatement.setString(column.getIndex() + 1, getPageReader().getString(column));
190
+ } catch (SQLException e) {
191
+ throw new RuntimeException(e);
192
+ }
193
+ }
194
+ };
195
+ }
196
+ }
@@ -0,0 +1,34 @@
1
+ package org.embulk.output.cdata.procedures;
2
+
3
+ import org.slf4j.Logger;
4
+ import org.slf4j.LoggerFactory;
5
+
6
+ import java.sql.Connection;
7
+ import java.sql.SQLException;
8
+ import java.sql.Statement;
9
+ import java.util.Arrays;
10
+
11
+ public class DeleteFile {
12
+
13
+ private static final Logger logger = LoggerFactory.getLogger(DeleteFile.class);
14
+ public static String[] SUPPOERTED_CLASS_NAMES = { "cdata.jdbc.csv.CSVDriver" };
15
+
16
+ public static void execute(Connection conn, String fileName) throws SQLException {
17
+ String className = conn.getMetaData().getDriverName();
18
+ if (Arrays.asList(SUPPOERTED_CLASS_NAMES).contains(className)) {
19
+ logger.info("not supported delete file className, skipped delete file: " + className);
20
+ return;
21
+ }
22
+ Statement stmt = conn.createStatement();
23
+ String source = fileName + ".csv";
24
+ String sql = "EXEC DELETEFILE @PATH='" + source + "'";
25
+ try {
26
+ boolean ret = stmt.execute(sql);
27
+ if (!ret) {
28
+ logger.info("failed to delete file: " + source);
29
+ }
30
+ } catch (Exception e) {
31
+ logger.error(e.getMessage());
32
+ }
33
+ }
34
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-cdata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - sampo02
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-07-06 00:00:00.000000000 Z
11
+ date: 2022-10-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,7 +49,7 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
- - classpath/embulk-output-cdata-0.3.4.jar
52
+ - classpath/embulk-output-cdata-0.4.1.jar
53
53
  - config/checkstyle/checkstyle.xml
54
54
  - config/checkstyle/default.xml
55
55
  - gradle/wrapper/gradle-wrapper.jar
@@ -58,8 +58,12 @@ files:
58
58
  - gradlew.bat
59
59
  - lib/embulk/output/cdata.rb
60
60
  - src/main/java/org/embulk/output/cdata/CDataOutputPlugin.java
61
+ - src/main/java/org/embulk/output/cdata/CDataPageOutputForInsert.java
62
+ - src/main/java/org/embulk/output/cdata/CDataPageOutputForManualUpsert.java
61
63
  - src/main/java/org/embulk/output/cdata/CDataPageOutputForUpdate.java
62
64
  - src/main/java/org/embulk/output/cdata/CDataPageOutputForUpsert.java
65
+ - src/main/java/org/embulk/output/cdata/CDataPageOutputForUpsertBase.java
66
+ - src/main/java/org/embulk/output/cdata/procedures/DeleteFile.java
63
67
  - src/test/java/org/embulk/output/cdata/TestCDataOutputPlugin.java
64
68
  homepage: https://github.com/datable-inc/embulk-output-cdata
65
69
  licenses: