embulk-output-postgresql 0.7.2 → 0.7.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +6 -2
- data/classpath/{embulk-output-jdbc-0.7.2.jar → embulk-output-jdbc-0.7.3.jar} +0 -0
- data/classpath/embulk-output-postgresql-0.7.3.jar +0 -0
- data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java +2 -2
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java +80 -0
- metadata +4 -4
- data/classpath/embulk-output-postgresql-0.7.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7acb9bf51cfa9656c136a38727e5730c142e8d14
|
4
|
+
data.tar.gz: 31cc497361027727ff1526049962e6772b65bb13
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80812f48c9dcb3530cb47a712b7c2b501d0859c6ada431ce4db4f109292baeb19f3dc274ada6e58bb7f58378d667997d95f47e887244d9c06cd22a377a78f82c
|
7
|
+
data.tar.gz: 6292455416b8487cdc84a4ad485ac351aeecf97656a7f1194dc15a0deda257f41cbda6f41160c95a618c213df44a32e5d47f2190eab8c079e9da0b4f470aee65
|
data/README.md
CHANGED
@@ -21,7 +21,7 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
|
|
21
21
|
- **retry_limit** max retry count for database operations (integer, default: 12)
|
22
22
|
- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
|
23
23
|
- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
|
24
|
-
- **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "
|
24
|
+
- **mode**: "insert", "insert_direct", "truncate_insert", "replace", "merge" or "merge_direct". See below. (string, required)
|
25
25
|
- **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
|
26
26
|
- **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = foo + S.foo` (`S` means source table). (string array, default: always overwrites with new values)
|
27
27
|
- **ssl**: enables SSL. data will be encrypted but CA or certification will not be verified (boolean, default: false)
|
@@ -53,9 +53,13 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
|
|
53
53
|
* Transactional: Yes.
|
54
54
|
* Resumable: No.
|
55
55
|
* **merge**:
|
56
|
-
* Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `
|
56
|
+
* Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `WITH updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
|
57
57
|
* Transactional: Yes.
|
58
58
|
* Resumable: Yes.
|
59
|
+
* **merge_direct**:
|
60
|
+
* Behavior: This mode inserts rows to the target table directly using `WITH S AS (SELECT ...), updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query. If the target table doesn't exist, it is created automatically.
|
61
|
+
* Transactional: No.
|
62
|
+
* Resumable: No.
|
59
63
|
|
60
64
|
### Supported types
|
61
65
|
|
Binary file
|
Binary file
|
@@ -64,7 +64,7 @@ public class PostgreSQLOutputPlugin
|
|
64
64
|
{
|
65
65
|
return new Features()
|
66
66
|
.setMaxTableNameLength(30)
|
67
|
-
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
67
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.MERGE_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
68
68
|
.setIgnoreMergeKeys(false);
|
69
69
|
}
|
70
70
|
|
@@ -111,7 +111,7 @@ public class PostgreSQLOutputPlugin
|
|
111
111
|
protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
|
112
112
|
{
|
113
113
|
if (mergeConfig.isPresent()) {
|
114
|
-
|
114
|
+
return new StandardBatchInsert(getConnector(task, true), mergeConfig);
|
115
115
|
}
|
116
116
|
return new PostgreSQLCopyBatchInsert(getConnector(task, true));
|
117
117
|
}
|
@@ -44,6 +44,86 @@ public class PostgreSQLOutputConnection
|
|
44
44
|
return new CopyManager((BaseConnection) connection);
|
45
45
|
}
|
46
46
|
|
47
|
+
@Override
|
48
|
+
protected String buildPreparedMergeSql(String toTable, JdbcSchema schema, MergeConfig mergeConfig) throws SQLException
|
49
|
+
{
|
50
|
+
StringBuilder sb = new StringBuilder();
|
51
|
+
|
52
|
+
sb.append("WITH S AS (");
|
53
|
+
sb.append("SELECT ");
|
54
|
+
for (int i = 0; i < schema.getCount(); i++) {
|
55
|
+
if (i != 0) { sb.append(", "); }
|
56
|
+
sb.append("? AS ");
|
57
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
58
|
+
}
|
59
|
+
sb.append("),");
|
60
|
+
sb.append("updated AS (");
|
61
|
+
sb.append("UPDATE ");
|
62
|
+
quoteIdentifierString(sb, toTable);
|
63
|
+
sb.append(" SET ");
|
64
|
+
if (mergeConfig.getMergeRule().isPresent()) {
|
65
|
+
List<String> rule = mergeConfig.getMergeRule().get();
|
66
|
+
for (int i = 0; i < rule.size(); i++) {
|
67
|
+
if (i != 0) { sb.append(", "); }
|
68
|
+
sb.append(rule.get(i));
|
69
|
+
}
|
70
|
+
} else {
|
71
|
+
for (int i = 0; i < schema.getCount(); i++) {
|
72
|
+
if (i != 0) { sb.append(", "); }
|
73
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
74
|
+
sb.append(" = S.");
|
75
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
76
|
+
}
|
77
|
+
}
|
78
|
+
sb.append(" FROM S");
|
79
|
+
sb.append(" WHERE ");
|
80
|
+
List<String> mergeKeys = mergeConfig.getMergeKeys();
|
81
|
+
for (int i = 0; i < mergeKeys.size(); i++) {
|
82
|
+
if (i != 0) { sb.append(" AND "); }
|
83
|
+
quoteIdentifierString(sb, toTable);
|
84
|
+
sb.append(".");
|
85
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
86
|
+
sb.append(" = ");
|
87
|
+
sb.append("S.");
|
88
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
89
|
+
}
|
90
|
+
sb.append(" RETURNING ");
|
91
|
+
for (int i = 0; i < mergeKeys.size(); i++) {
|
92
|
+
if (i != 0) { sb.append(", "); }
|
93
|
+
sb.append("S.");
|
94
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
95
|
+
}
|
96
|
+
sb.append(") ");
|
97
|
+
|
98
|
+
sb.append("INSERT INTO ");
|
99
|
+
quoteIdentifierString(sb, toTable);
|
100
|
+
sb.append(" (");
|
101
|
+
for (int i = 0; i < schema.getCount(); i++) {
|
102
|
+
if (i != 0) { sb.append(", "); }
|
103
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
104
|
+
}
|
105
|
+
sb.append(") ");
|
106
|
+
sb.append("SELECT ");
|
107
|
+
for(int i = 0; i < schema.getCount(); i++) {
|
108
|
+
if (i != 0) { sb.append(", "); }
|
109
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
110
|
+
}
|
111
|
+
sb.append(" FROM S ");
|
112
|
+
sb.append("WHERE NOT EXISTS (");
|
113
|
+
sb.append("SELECT 1 FROM updated WHERE ");
|
114
|
+
for (int i = 0; i < mergeKeys.size(); i++) {
|
115
|
+
if (i != 0) { sb.append(" AND "); }
|
116
|
+
sb.append("S.");
|
117
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
118
|
+
sb.append(" = ");
|
119
|
+
sb.append("updated.");
|
120
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
121
|
+
}
|
122
|
+
sb.append(") ");
|
123
|
+
|
124
|
+
return sb.toString();
|
125
|
+
}
|
126
|
+
|
47
127
|
@Override
|
48
128
|
protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
|
49
129
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-06 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
23
|
-
- classpath/embulk-output-postgresql-0.7.
|
22
|
+
- classpath/embulk-output-jdbc-0.7.3.jar
|
23
|
+
- classpath/embulk-output-postgresql-0.7.3.jar
|
24
24
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
25
25
|
- lib/embulk/output/postgresql.rb
|
26
26
|
- src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java
|
Binary file
|