embulk-output-postgresql 0.4.0 → 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +89 -89
- data/build.gradle +7 -7
- data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
- data/classpath/{embulk-output-postgresql-0.4.0.jar → embulk-output-postgresql-0.4.1.jar} +0 -0
- data/lib/embulk/output/postgresql.rb +3 -3
- data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java +165 -165
- data/src/main/java/org/embulk/output/postgresql/AbstractPostgreSQLCopyBatchInsert.java +245 -245
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java +73 -73
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java +157 -157
- data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java +40 -40
- metadata +4 -4
@@ -1,157 +1,157 @@
|
|
1
|
-
package org.embulk.output.postgresql;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import java.sql.Connection;
|
5
|
-
import java.sql.SQLException;
|
6
|
-
import java.sql.Statement;
|
7
|
-
import org.postgresql.copy.CopyManager;
|
8
|
-
import org.postgresql.core.BaseConnection;
|
9
|
-
import org.embulk.spi.Exec;
|
10
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
11
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
12
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
13
|
-
|
14
|
-
public class PostgreSQLOutputConnection
|
15
|
-
extends JdbcOutputConnection
|
16
|
-
{
|
17
|
-
public PostgreSQLOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
-
throws SQLException
|
19
|
-
{
|
20
|
-
super(connection, schemaName);
|
21
|
-
connection.setAutoCommit(autoCommit);
|
22
|
-
}
|
23
|
-
|
24
|
-
public String buildCopySql(String toTable, JdbcSchema toTableSchema)
|
25
|
-
{
|
26
|
-
StringBuilder sb = new StringBuilder();
|
27
|
-
|
28
|
-
sb.append("COPY ");
|
29
|
-
quoteIdentifierString(sb, toTable);
|
30
|
-
sb.append(" (");
|
31
|
-
for (int i=0; i < toTableSchema.getCount(); i++) {
|
32
|
-
if (i != 0) { sb.append(", "); }
|
33
|
-
quoteIdentifierString(sb, toTableSchema.getColumnName(i));
|
34
|
-
}
|
35
|
-
sb.append(") ");
|
36
|
-
sb.append("FROM STDIN");
|
37
|
-
|
38
|
-
return sb.toString();
|
39
|
-
}
|
40
|
-
|
41
|
-
public CopyManager newCopyManager() throws SQLException
|
42
|
-
{
|
43
|
-
return new CopyManager((BaseConnection) connection);
|
44
|
-
}
|
45
|
-
|
46
|
-
@Override
|
47
|
-
protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
|
48
|
-
{
|
49
|
-
StringBuilder sb = new StringBuilder();
|
50
|
-
|
51
|
-
sb.append("WITH updated AS (");
|
52
|
-
sb.append("UPDATE ");
|
53
|
-
quoteIdentifierString(sb, toTable);
|
54
|
-
sb.append(" SET ");
|
55
|
-
for (int i=0; i < schema.getCount(); i++) {
|
56
|
-
if (i != 0) { sb.append(", "); }
|
57
|
-
quoteIdentifierString(sb, schema.getColumnName(i));
|
58
|
-
sb.append(" = S.");
|
59
|
-
quoteIdentifierString(sb, schema.getColumnName(i));
|
60
|
-
}
|
61
|
-
sb.append(" FROM (");
|
62
|
-
for (int i=0; i < fromTables.size(); i++) {
|
63
|
-
if (i != 0) { sb.append(" UNION ALL "); }
|
64
|
-
sb.append("SELECT ");
|
65
|
-
for(int j=0; j < schema.getCount(); j++) {
|
66
|
-
if (j != 0) { sb.append(", "); }
|
67
|
-
quoteIdentifierString(sb, schema.getColumnName(j));
|
68
|
-
}
|
69
|
-
sb.append(" FROM ");
|
70
|
-
quoteIdentifierString(sb, fromTables.get(i));
|
71
|
-
}
|
72
|
-
sb.append(") S");
|
73
|
-
sb.append(" WHERE ");
|
74
|
-
for (int i=0; i < mergeKeys.size(); i++) {
|
75
|
-
if (i != 0) { sb.append(" AND "); }
|
76
|
-
quoteIdentifierString(sb, toTable);
|
77
|
-
sb.append(".");
|
78
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
79
|
-
sb.append(" = ");
|
80
|
-
sb.append("S.");
|
81
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
82
|
-
}
|
83
|
-
sb.append(" RETURNING ");
|
84
|
-
for (int i=0; i < mergeKeys.size(); i++) {
|
85
|
-
if (i != 0) { sb.append(", "); }
|
86
|
-
sb.append("S.");
|
87
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
88
|
-
}
|
89
|
-
sb.append(") ");
|
90
|
-
|
91
|
-
sb.append("INSERT INTO ");
|
92
|
-
quoteIdentifierString(sb, toTable);
|
93
|
-
sb.append(" (");
|
94
|
-
for (int i=0; i < schema.getCount(); i++) {
|
95
|
-
if (i != 0) { sb.append(", "); }
|
96
|
-
quoteIdentifierString(sb, schema.getColumnName(i));
|
97
|
-
}
|
98
|
-
sb.append(") ");
|
99
|
-
sb.append("SELECT DISTINCT ON (");
|
100
|
-
for (int i=0; i < mergeKeys.size(); i++) {
|
101
|
-
if (i != 0) { sb.append(", "); }
|
102
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
103
|
-
}
|
104
|
-
sb.append(") * FROM (");
|
105
|
-
for (int i=0; i < fromTables.size(); i++) {
|
106
|
-
if (i != 0) { sb.append(" UNION ALL "); }
|
107
|
-
sb.append("SELECT ");
|
108
|
-
for(int j=0; j < schema.getCount(); j++) {
|
109
|
-
if (j != 0) { sb.append(", "); }
|
110
|
-
quoteIdentifierString(sb, schema.getColumnName(j));
|
111
|
-
}
|
112
|
-
sb.append(" FROM ");
|
113
|
-
quoteIdentifierString(sb, fromTables.get(i));
|
114
|
-
}
|
115
|
-
sb.append(") S ");
|
116
|
-
sb.append("WHERE NOT EXISTS (");
|
117
|
-
sb.append("SELECT 1 FROM updated WHERE ");
|
118
|
-
for (int i=0; i < mergeKeys.size(); i++) {
|
119
|
-
if (i != 0) { sb.append(" AND "); }
|
120
|
-
sb.append("S.");
|
121
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
122
|
-
sb.append(" = ");
|
123
|
-
sb.append("updated.");
|
124
|
-
quoteIdentifierString(sb, mergeKeys.get(i));
|
125
|
-
}
|
126
|
-
sb.append(") ");
|
127
|
-
|
128
|
-
return sb.toString();
|
129
|
-
}
|
130
|
-
|
131
|
-
protected void collectReplaceView(List<String> fromTables, JdbcSchema schema, String toTable) throws SQLException
|
132
|
-
{
|
133
|
-
Statement stmt = connection.createStatement();
|
134
|
-
try {
|
135
|
-
String sql = buildCollectInsertSql(fromTables, schema, toTable);
|
136
|
-
executeUpdate(stmt, sql);
|
137
|
-
commitIfNecessary(connection);
|
138
|
-
} catch (SQLException ex) {
|
139
|
-
throw safeRollback(connection, ex);
|
140
|
-
} finally {
|
141
|
-
stmt.close();
|
142
|
-
}
|
143
|
-
}
|
144
|
-
|
145
|
-
@Override
|
146
|
-
protected String buildColumnTypeName(JdbcColumn c)
|
147
|
-
{
|
148
|
-
switch(c.getSimpleTypeName()) {
|
149
|
-
case "CLOB":
|
150
|
-
return "TEXT";
|
151
|
-
case "BLOB":
|
152
|
-
return "BYTEA";
|
153
|
-
default:
|
154
|
-
return super.buildColumnTypeName(c);
|
155
|
-
}
|
156
|
-
}
|
157
|
-
}
|
1
|
+
package org.embulk.output.postgresql;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.sql.Connection;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import java.sql.Statement;
|
7
|
+
import org.postgresql.copy.CopyManager;
|
8
|
+
import org.postgresql.core.BaseConnection;
|
9
|
+
import org.embulk.spi.Exec;
|
10
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
11
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
12
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
13
|
+
|
14
|
+
public class PostgreSQLOutputConnection
|
15
|
+
extends JdbcOutputConnection
|
16
|
+
{
|
17
|
+
public PostgreSQLOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
+
throws SQLException
|
19
|
+
{
|
20
|
+
super(connection, schemaName);
|
21
|
+
connection.setAutoCommit(autoCommit);
|
22
|
+
}
|
23
|
+
|
24
|
+
public String buildCopySql(String toTable, JdbcSchema toTableSchema)
|
25
|
+
{
|
26
|
+
StringBuilder sb = new StringBuilder();
|
27
|
+
|
28
|
+
sb.append("COPY ");
|
29
|
+
quoteIdentifierString(sb, toTable);
|
30
|
+
sb.append(" (");
|
31
|
+
for (int i=0; i < toTableSchema.getCount(); i++) {
|
32
|
+
if (i != 0) { sb.append(", "); }
|
33
|
+
quoteIdentifierString(sb, toTableSchema.getColumnName(i));
|
34
|
+
}
|
35
|
+
sb.append(") ");
|
36
|
+
sb.append("FROM STDIN");
|
37
|
+
|
38
|
+
return sb.toString();
|
39
|
+
}
|
40
|
+
|
41
|
+
public CopyManager newCopyManager() throws SQLException
|
42
|
+
{
|
43
|
+
return new CopyManager((BaseConnection) connection);
|
44
|
+
}
|
45
|
+
|
46
|
+
@Override
|
47
|
+
protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
|
48
|
+
{
|
49
|
+
StringBuilder sb = new StringBuilder();
|
50
|
+
|
51
|
+
sb.append("WITH updated AS (");
|
52
|
+
sb.append("UPDATE ");
|
53
|
+
quoteIdentifierString(sb, toTable);
|
54
|
+
sb.append(" SET ");
|
55
|
+
for (int i=0; i < schema.getCount(); i++) {
|
56
|
+
if (i != 0) { sb.append(", "); }
|
57
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
58
|
+
sb.append(" = S.");
|
59
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
60
|
+
}
|
61
|
+
sb.append(" FROM (");
|
62
|
+
for (int i=0; i < fromTables.size(); i++) {
|
63
|
+
if (i != 0) { sb.append(" UNION ALL "); }
|
64
|
+
sb.append("SELECT ");
|
65
|
+
for(int j=0; j < schema.getCount(); j++) {
|
66
|
+
if (j != 0) { sb.append(", "); }
|
67
|
+
quoteIdentifierString(sb, schema.getColumnName(j));
|
68
|
+
}
|
69
|
+
sb.append(" FROM ");
|
70
|
+
quoteIdentifierString(sb, fromTables.get(i));
|
71
|
+
}
|
72
|
+
sb.append(") S");
|
73
|
+
sb.append(" WHERE ");
|
74
|
+
for (int i=0; i < mergeKeys.size(); i++) {
|
75
|
+
if (i != 0) { sb.append(" AND "); }
|
76
|
+
quoteIdentifierString(sb, toTable);
|
77
|
+
sb.append(".");
|
78
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
79
|
+
sb.append(" = ");
|
80
|
+
sb.append("S.");
|
81
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
82
|
+
}
|
83
|
+
sb.append(" RETURNING ");
|
84
|
+
for (int i=0; i < mergeKeys.size(); i++) {
|
85
|
+
if (i != 0) { sb.append(", "); }
|
86
|
+
sb.append("S.");
|
87
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
88
|
+
}
|
89
|
+
sb.append(") ");
|
90
|
+
|
91
|
+
sb.append("INSERT INTO ");
|
92
|
+
quoteIdentifierString(sb, toTable);
|
93
|
+
sb.append(" (");
|
94
|
+
for (int i=0; i < schema.getCount(); i++) {
|
95
|
+
if (i != 0) { sb.append(", "); }
|
96
|
+
quoteIdentifierString(sb, schema.getColumnName(i));
|
97
|
+
}
|
98
|
+
sb.append(") ");
|
99
|
+
sb.append("SELECT DISTINCT ON (");
|
100
|
+
for (int i=0; i < mergeKeys.size(); i++) {
|
101
|
+
if (i != 0) { sb.append(", "); }
|
102
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
103
|
+
}
|
104
|
+
sb.append(") * FROM (");
|
105
|
+
for (int i=0; i < fromTables.size(); i++) {
|
106
|
+
if (i != 0) { sb.append(" UNION ALL "); }
|
107
|
+
sb.append("SELECT ");
|
108
|
+
for(int j=0; j < schema.getCount(); j++) {
|
109
|
+
if (j != 0) { sb.append(", "); }
|
110
|
+
quoteIdentifierString(sb, schema.getColumnName(j));
|
111
|
+
}
|
112
|
+
sb.append(" FROM ");
|
113
|
+
quoteIdentifierString(sb, fromTables.get(i));
|
114
|
+
}
|
115
|
+
sb.append(") S ");
|
116
|
+
sb.append("WHERE NOT EXISTS (");
|
117
|
+
sb.append("SELECT 1 FROM updated WHERE ");
|
118
|
+
for (int i=0; i < mergeKeys.size(); i++) {
|
119
|
+
if (i != 0) { sb.append(" AND "); }
|
120
|
+
sb.append("S.");
|
121
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
122
|
+
sb.append(" = ");
|
123
|
+
sb.append("updated.");
|
124
|
+
quoteIdentifierString(sb, mergeKeys.get(i));
|
125
|
+
}
|
126
|
+
sb.append(") ");
|
127
|
+
|
128
|
+
return sb.toString();
|
129
|
+
}
|
130
|
+
|
131
|
+
protected void collectReplaceView(List<String> fromTables, JdbcSchema schema, String toTable) throws SQLException
|
132
|
+
{
|
133
|
+
Statement stmt = connection.createStatement();
|
134
|
+
try {
|
135
|
+
String sql = buildCollectInsertSql(fromTables, schema, toTable);
|
136
|
+
executeUpdate(stmt, sql);
|
137
|
+
commitIfNecessary(connection);
|
138
|
+
} catch (SQLException ex) {
|
139
|
+
throw safeRollback(connection, ex);
|
140
|
+
} finally {
|
141
|
+
stmt.close();
|
142
|
+
}
|
143
|
+
}
|
144
|
+
|
145
|
+
@Override
|
146
|
+
protected String buildColumnTypeName(JdbcColumn c)
|
147
|
+
{
|
148
|
+
switch(c.getSimpleTypeName()) {
|
149
|
+
case "CLOB":
|
150
|
+
return "TEXT";
|
151
|
+
case "BLOB":
|
152
|
+
return "BYTEA";
|
153
|
+
default:
|
154
|
+
return super.buildColumnTypeName(c);
|
155
|
+
}
|
156
|
+
}
|
157
|
+
}
|
@@ -1,40 +1,40 @@
|
|
1
|
-
package org.embulk.output.postgresql;
|
2
|
-
|
3
|
-
import java.util.Properties;
|
4
|
-
import java.sql.Driver;
|
5
|
-
import java.sql.Connection;
|
6
|
-
import java.sql.SQLException;
|
7
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
|
10
|
-
public class PostgreSQLOutputConnector
|
11
|
-
implements JdbcOutputConnector
|
12
|
-
{
|
13
|
-
private static final Driver driver = new org.postgresql.Driver();
|
14
|
-
|
15
|
-
private final String url;
|
16
|
-
private final Properties properties;
|
17
|
-
private final String schemaName;
|
18
|
-
|
19
|
-
public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
|
20
|
-
{
|
21
|
-
this.url = url;
|
22
|
-
this.properties = properties;
|
23
|
-
this.schemaName = schemaName;
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
-
{
|
29
|
-
Connection c = driver.connect(url, properties);
|
30
|
-
try {
|
31
|
-
PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
|
32
|
-
c = null;
|
33
|
-
return con;
|
34
|
-
} finally {
|
35
|
-
if (c != null) {
|
36
|
-
c.close();
|
37
|
-
}
|
38
|
-
}
|
39
|
-
}
|
40
|
-
}
|
1
|
+
package org.embulk.output.postgresql;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.sql.Driver;
|
5
|
+
import java.sql.Connection;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
|
10
|
+
public class PostgreSQLOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private static final Driver driver = new org.postgresql.Driver();
|
14
|
+
|
15
|
+
private final String url;
|
16
|
+
private final Properties properties;
|
17
|
+
private final String schemaName;
|
18
|
+
|
19
|
+
public PostgreSQLOutputConnector(String url, Properties properties, String schemaName)
|
20
|
+
{
|
21
|
+
this.url = url;
|
22
|
+
this.properties = properties;
|
23
|
+
this.schemaName = schemaName;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public PostgreSQLOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
+
{
|
29
|
+
Connection c = driver.connect(url, properties);
|
30
|
+
try {
|
31
|
+
PostgreSQLOutputConnection con = new PostgreSQLOutputConnection(c, schemaName, autoCommit);
|
32
|
+
c = null;
|
33
|
+
return con;
|
34
|
+
} finally {
|
35
|
+
if (c != null) {
|
36
|
+
c.close();
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-postgresql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-06
|
11
|
+
date: 2015-07-06 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -25,8 +25,8 @@ files:
|
|
25
25
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLCopyBatchInsert.java
|
26
26
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
|
27
27
|
- src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
|
28
|
-
- classpath/embulk-output-jdbc-0.4.
|
29
|
-
- classpath/embulk-output-postgresql-0.4.
|
28
|
+
- classpath/embulk-output-jdbc-0.4.1.jar
|
29
|
+
- classpath/embulk-output-postgresql-0.4.1.jar
|
30
30
|
- classpath/jna-4.1.0.jar
|
31
31
|
- classpath/jna-platform-4.1.0.jar
|
32
32
|
- classpath/postgresql-9.4-1200-jdbc41.jar
|