embulk-output-postgresql 0.7.8 → 0.7.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -0
- data/classpath/embulk-output-jdbc-0.7.9.jar +0 -0
- data/classpath/embulk-output-postgresql-0.7.9.jar +0 -0
- data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java +14 -0
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java +7 -5
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java +31 -17
- metadata +4 -4
- data/classpath/embulk-output-jdbc-0.7.8.jar +0 -0
- data/classpath/embulk-output-postgresql-0.7.8.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 92ad15e7a7d79e5584efbc486e48bafe4eb2096f
|
4
|
+
data.tar.gz: 4c6731253c58f888fded75e7ee0588234bd7b123
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12e94ac98087761e4c996895a01d5be30d529efc140f593823280829f1f32bc08864f4b38e1638ded04bb5d03c62e154e1e4ba0e4ba5b8756bbe46043cb5e1a2
|
7
|
+
data.tar.gz: da5fa17bdde718404b2331212da4c6dbffb2dc84de3f765ecc88e95a7af7e6432605eeb9f72e08012fdb1305860a20efa557b71cf875be282385e596cecfb986
|
data/README.md
CHANGED
@@ -16,6 +16,7 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
|
|
16
16
|
- **password**: database login password (string, default: "")
|
17
17
|
- **database**: destination database name (string, required)
|
18
18
|
- **schema**: destination schema name (string, default: "public")
|
19
|
+
- **temp_schema**: schema name for intermediate tables. by default, intermediate tables will be created in the schema specified by `schema`. replace mode doesn't support temp_schema. (string, optional)
|
19
20
|
- **table**: destination table name (string, required)
|
20
21
|
- **options**: extra connection properties (hash, default: {})
|
21
22
|
- **retry_limit** max retry count for database operations (integer, default: 12)
|
Binary file
|
Binary file
|
@@ -48,6 +48,10 @@ public class PostgreSQLOutputPlugin
|
|
48
48
|
@ConfigDefault("\"public\"")
|
49
49
|
public String getSchema();
|
50
50
|
|
51
|
+
@Config("temp_schema")
|
52
|
+
@ConfigDefault("null")
|
53
|
+
public Optional<String> getTempSchema();
|
54
|
+
|
51
55
|
@Config("ssl")
|
52
56
|
@ConfigDefault("false")
|
53
57
|
public boolean getSsl();
|
@@ -107,6 +111,16 @@ public class PostgreSQLOutputPlugin
|
|
107
111
|
return new PostgreSQLOutputConnector(url, props, t.getSchema());
|
108
112
|
}
|
109
113
|
|
114
|
+
@Override
|
115
|
+
protected TableIdentifier buildIntermediateTableId(JdbcOutputConnection con, PluginTask task, String tableName) {
|
116
|
+
PostgreSQLPluginTask t = (PostgreSQLPluginTask) task;
|
117
|
+
// replace mode doesn't support temp_schema because ALTER TABLE cannot change schema of table
|
118
|
+
if (t.getTempSchema().isPresent() && t.getMode() != Mode.REPLACE) {
|
119
|
+
return new TableIdentifier(null, t.getTempSchema().get(), tableName);
|
120
|
+
}
|
121
|
+
return super.buildIntermediateTableId(con, task, tableName);
|
122
|
+
}
|
123
|
+
|
110
124
|
@Override
|
111
125
|
protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
|
112
126
|
{
|
@@ -1,13 +1,15 @@
|
|
1
1
|
package org.embulk.output.postgresql;
|
2
2
|
|
3
3
|
import java.io.File;
|
4
|
-
import java.io.IOException;
|
5
4
|
import java.io.FileInputStream;
|
5
|
+
import java.io.IOException;
|
6
6
|
import java.sql.SQLException;
|
7
|
-
|
8
|
-
import org.postgresql.copy.CopyManager;
|
9
|
-
import org.embulk.spi.Exec;
|
7
|
+
|
10
8
|
import org.embulk.output.jdbc.JdbcSchema;
|
9
|
+
import org.embulk.output.jdbc.TableIdentifier;
|
10
|
+
import org.embulk.spi.Exec;
|
11
|
+
import org.postgresql.copy.CopyManager;
|
12
|
+
import org.slf4j.Logger;
|
11
13
|
|
12
14
|
public class PostgreSQLCopyBatchInsert
|
13
15
|
extends AbstractPostgreSQLCopyBatchInsert
|
@@ -27,7 +29,7 @@ public class PostgreSQLCopyBatchInsert
|
|
27
29
|
}
|
28
30
|
|
29
31
|
@Override
|
30
|
-
public void prepare(
|
32
|
+
public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
|
31
33
|
{
|
32
34
|
this.connection = connector.connect(true);
|
33
35
|
this.copySql = connection.buildCopySql(loadTable, insertSchema);
|
@@ -1,16 +1,17 @@
|
|
1
1
|
package org.embulk.output.postgresql;
|
2
2
|
|
3
|
-
import java.util.List;
|
4
3
|
import java.sql.Connection;
|
5
4
|
import java.sql.SQLException;
|
6
5
|
import java.sql.Statement;
|
6
|
+
import java.util.List;
|
7
7
|
|
8
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
9
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
10
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
8
11
|
import org.embulk.output.jdbc.MergeConfig;
|
12
|
+
import org.embulk.output.jdbc.TableIdentifier;
|
9
13
|
import org.postgresql.copy.CopyManager;
|
10
14
|
import org.postgresql.core.BaseConnection;
|
11
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
12
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
13
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
14
15
|
|
15
16
|
public class PostgreSQLOutputConnection
|
16
17
|
extends JdbcOutputConnection
|
@@ -24,12 +25,12 @@ public class PostgreSQLOutputConnection
|
|
24
25
|
connection.setAutoCommit(autoCommit);
|
25
26
|
}
|
26
27
|
|
27
|
-
public String buildCopySql(
|
28
|
+
public String buildCopySql(TableIdentifier toTable, JdbcSchema toTableSchema)
|
28
29
|
{
|
29
30
|
StringBuilder sb = new StringBuilder();
|
30
31
|
|
31
32
|
sb.append("COPY ");
|
32
|
-
|
33
|
+
quoteTableIdentifier(sb, toTable);
|
33
34
|
sb.append(" (");
|
34
35
|
for (int i = 0; i < toTableSchema.getCount(); i++) {
|
35
36
|
if (i != 0) { sb.append(", "); }
|
@@ -47,7 +48,7 @@ public class PostgreSQLOutputConnection
|
|
47
48
|
}
|
48
49
|
|
49
50
|
@Override
|
50
|
-
protected String buildPreparedMergeSql(
|
51
|
+
protected String buildPreparedMergeSql(TableIdentifier toTable, JdbcSchema schema, MergeConfig mergeConfig) throws SQLException
|
51
52
|
{
|
52
53
|
StringBuilder sb = new StringBuilder();
|
53
54
|
|
@@ -62,7 +63,7 @@ public class PostgreSQLOutputConnection
|
|
62
63
|
sb.append("),");
|
63
64
|
sb.append("updated AS (");
|
64
65
|
sb.append("UPDATE ");
|
65
|
-
|
66
|
+
quoteTableIdentifier(sb, toTable);
|
66
67
|
sb.append(" SET ");
|
67
68
|
if (mergeConfig.getMergeRule().isPresent()) {
|
68
69
|
List<String> rule = mergeConfig.getMergeRule().get();
|
@@ -83,7 +84,7 @@ public class PostgreSQLOutputConnection
|
|
83
84
|
List<String> mergeKeys = mergeConfig.getMergeKeys();
|
84
85
|
for (int i = 0; i < mergeKeys.size(); i++) {
|
85
86
|
if (i != 0) { sb.append(" AND "); }
|
86
|
-
|
87
|
+
quoteTableIdentifier(sb, toTable);
|
87
88
|
sb.append(".");
|
88
89
|
quoteIdentifierString(sb, mergeKeys.get(i));
|
89
90
|
sb.append(" = ");
|
@@ -99,7 +100,7 @@ public class PostgreSQLOutputConnection
|
|
99
100
|
sb.append(") ");
|
100
101
|
|
101
102
|
sb.append("INSERT INTO ");
|
102
|
-
|
103
|
+
quoteTableIdentifier(sb, toTable);
|
103
104
|
sb.append(" (");
|
104
105
|
for (int i = 0; i < schema.getCount(); i++) {
|
105
106
|
if (i != 0) { sb.append(", "); }
|
@@ -128,13 +129,13 @@ public class PostgreSQLOutputConnection
|
|
128
129
|
}
|
129
130
|
|
130
131
|
@Override
|
131
|
-
protected String buildCollectMergeSql(List<
|
132
|
+
protected String buildCollectMergeSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig) throws SQLException
|
132
133
|
{
|
133
134
|
StringBuilder sb = new StringBuilder();
|
134
135
|
|
135
136
|
sb.append("WITH updated AS (");
|
136
137
|
sb.append("UPDATE ");
|
137
|
-
|
138
|
+
quoteTableIdentifier(sb, toTable);
|
138
139
|
sb.append(" SET ");
|
139
140
|
if (mergeConfig.getMergeRule().isPresent()) {
|
140
141
|
List<String> rule = mergeConfig.getMergeRule().get();
|
@@ -163,14 +164,14 @@ public class PostgreSQLOutputConnection
|
|
163
164
|
quoteIdentifierString(sb, schema.getColumnName(j));
|
164
165
|
}
|
165
166
|
sb.append(" FROM ");
|
166
|
-
|
167
|
+
quoteTableIdentifier(sb, fromTables.get(i));
|
167
168
|
}
|
168
169
|
sb.append(") S");
|
169
170
|
sb.append(" WHERE ");
|
170
171
|
List<String> mergeKeys = mergeConfig.getMergeKeys();
|
171
172
|
for (int i = 0; i < mergeKeys.size(); i++) {
|
172
173
|
if (i != 0) { sb.append(" AND "); }
|
173
|
-
|
174
|
+
quoteTableIdentifier(sb, toTable);
|
174
175
|
sb.append(".");
|
175
176
|
quoteIdentifierString(sb, mergeKeys.get(i));
|
176
177
|
sb.append(" = ");
|
@@ -186,7 +187,7 @@ public class PostgreSQLOutputConnection
|
|
186
187
|
sb.append(") ");
|
187
188
|
|
188
189
|
sb.append("INSERT INTO ");
|
189
|
-
|
190
|
+
quoteTableIdentifier(sb, toTable);
|
190
191
|
sb.append(" (");
|
191
192
|
for (int i = 0; i < schema.getCount(); i++) {
|
192
193
|
if (i != 0) { sb.append(", "); }
|
@@ -207,7 +208,7 @@ public class PostgreSQLOutputConnection
|
|
207
208
|
quoteIdentifierString(sb, schema.getColumnName(j));
|
208
209
|
}
|
209
210
|
sb.append(" FROM ");
|
210
|
-
|
211
|
+
quoteTableIdentifier(sb, fromTables.get(i));
|
211
212
|
}
|
212
213
|
sb.append(") S ");
|
213
214
|
sb.append("WHERE NOT EXISTS (");
|
@@ -225,7 +226,7 @@ public class PostgreSQLOutputConnection
|
|
225
226
|
return sb.toString();
|
226
227
|
}
|
227
228
|
|
228
|
-
protected void collectReplaceView(List<
|
229
|
+
protected void collectReplaceView(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable) throws SQLException
|
229
230
|
{
|
230
231
|
Statement stmt = connection.createStatement();
|
231
232
|
try {
|
@@ -266,4 +267,17 @@ public class PostgreSQLOutputConnection
|
|
266
267
|
}
|
267
268
|
return super.buildColumnTypeName(c);
|
268
269
|
}
|
270
|
+
|
271
|
+
@Override
|
272
|
+
protected String buildRenameTableSql(TableIdentifier fromTable, TableIdentifier toTable)
|
273
|
+
{
|
274
|
+
// ALTER TABLE cannot change schema of table
|
275
|
+
StringBuilder sb = new StringBuilder();
|
276
|
+
sb.append("ALTER TABLE ");
|
277
|
+
quoteTableIdentifier(sb, fromTable);
|
278
|
+
sb.append(" RENAME TO ");
|
279
|
+
quoteIdentifierString(sb, toTable.getTableName());
|
280
|
+
return sb.toString();
|
281
|
+
}
|
282
|
+
|
269
283
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -19,8 +19,8 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
23
|
-
- classpath/embulk-output-postgresql-0.7.
|
22
|
+
- classpath/embulk-output-jdbc-0.7.9.jar
|
23
|
+
- classpath/embulk-output-postgresql-0.7.9.jar
|
24
24
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
25
25
|
- lib/embulk/output/postgresql.rb
|
26
26
|
- src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java
|
Binary file
|
Binary file
|