embulk-output-redshift 0.4.1 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +104 -104
- data/build.gradle +9 -9
- data/classpath/{aws-java-sdk-core-1.9.17.jar → aws-java-sdk-core-1.10.33.jar} +0 -0
- data/classpath/aws-java-sdk-kms-1.10.33.jar +0 -0
- data/classpath/aws-java-sdk-s3-1.10.33.jar +0 -0
- data/classpath/aws-java-sdk-sts-1.10.33.jar +0 -0
- data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
- data/classpath/embulk-output-postgresql-0.4.2.jar +0 -0
- data/classpath/{embulk-output-redshift-0.4.1.jar → embulk-output-redshift-0.4.2.jar} +0 -0
- data/classpath/{httpclient-4.3.4.jar → httpclient-4.3.6.jar} +0 -0
- data/classpath/{httpcore-4.3.2.jar → httpcore-4.3.3.jar} +0 -0
- data/classpath/postgresql-9.4-1205-jdbc41.jar +0 -0
- data/lib/embulk/output/redshift.rb +3 -3
- data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java +151 -151
- data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java +218 -218
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java +122 -122
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java +40 -40
- metadata +12 -17
- data/classpath/aws-java-sdk-kms-1.9.17.jar +0 -0
- data/classpath/aws-java-sdk-s3-1.9.17.jar +0 -0
- data/classpath/aws-java-sdk-sts-1.9.17.jar +0 -0
- data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
- data/classpath/embulk-output-postgresql-0.4.1.jar +0 -0
- data/classpath/jna-4.1.0.jar +0 -0
- data/classpath/jna-platform-4.1.0.jar +0 -0
- data/classpath/joda-time-2.8.1.jar +0 -0
- data/classpath/postgresql-9.4-1200-jdbc41.jar +0 -0
- data/classpath/slf4j-simple-1.7.7.jar +0 -0
- data/classpath/waffle-jna-1.7.jar +0 -0
@@ -1,122 +1,122 @@
|
|
1
|
-
package org.embulk.output.redshift;
|
2
|
-
|
3
|
-
import java.sql.Connection;
|
4
|
-
import java.sql.SQLException;
|
5
|
-
import java.sql.Statement;
|
6
|
-
import org.slf4j.Logger;
|
7
|
-
import org.embulk.spi.Exec;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
10
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
11
|
-
|
12
|
-
public class RedshiftOutputConnection
|
13
|
-
extends JdbcOutputConnection
|
14
|
-
{
|
15
|
-
private final Logger logger = Exec.getLogger(RedshiftOutputConnection.class);
|
16
|
-
|
17
|
-
public RedshiftOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
-
throws SQLException
|
19
|
-
{
|
20
|
-
super(connection, schemaName);
|
21
|
-
connection.setAutoCommit(autoCommit);
|
22
|
-
}
|
23
|
-
|
24
|
-
// Redshift does not support DROP TABLE IF EXISTS.
|
25
|
-
// Here runs DROP TABLE and ignores errors.
|
26
|
-
@Override
|
27
|
-
public void dropTableIfExists(String tableName) throws SQLException
|
28
|
-
{
|
29
|
-
Statement stmt = connection.createStatement();
|
30
|
-
try {
|
31
|
-
String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
|
32
|
-
executeUpdate(stmt, sql);
|
33
|
-
commitIfNecessary(connection);
|
34
|
-
} catch (SQLException ex) {
|
35
|
-
// ignore errors.
|
36
|
-
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
37
|
-
SQLException ignored = safeRollback(connection, ex);
|
38
|
-
} finally {
|
39
|
-
stmt.close();
|
40
|
-
}
|
41
|
-
}
|
42
|
-
|
43
|
-
// Redshift does not support DROP TABLE IF EXISTS.
|
44
|
-
// Dropping part runs DROP TABLE and ignores errors.
|
45
|
-
@Override
|
46
|
-
public void replaceTable(String fromTable, JdbcSchema schema, String toTable) throws SQLException
|
47
|
-
{
|
48
|
-
Statement stmt = connection.createStatement();
|
49
|
-
try {
|
50
|
-
try {
|
51
|
-
StringBuilder sb = new StringBuilder();
|
52
|
-
sb.append("DROP TABLE ");
|
53
|
-
quoteIdentifierString(sb, toTable);
|
54
|
-
String sql = sb.toString();
|
55
|
-
executeUpdate(stmt, sql);
|
56
|
-
} catch (SQLException ex) {
|
57
|
-
// ignore errors.
|
58
|
-
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
59
|
-
// rollback or comimt is required to recover failed transaction
|
60
|
-
SQLException ignored = safeRollback(connection, ex);
|
61
|
-
}
|
62
|
-
|
63
|
-
{
|
64
|
-
StringBuilder sb = new StringBuilder();
|
65
|
-
sb.append("ALTER TABLE ");
|
66
|
-
quoteIdentifierString(sb, fromTable);
|
67
|
-
sb.append(" RENAME TO ");
|
68
|
-
quoteIdentifierString(sb, toTable);
|
69
|
-
String sql = sb.toString();
|
70
|
-
executeUpdate(stmt, sql);
|
71
|
-
}
|
72
|
-
|
73
|
-
commitIfNecessary(connection);
|
74
|
-
} catch (SQLException ex) {
|
75
|
-
throw safeRollback(connection, ex);
|
76
|
-
} finally {
|
77
|
-
stmt.close();
|
78
|
-
}
|
79
|
-
}
|
80
|
-
|
81
|
-
@Override
|
82
|
-
protected String buildColumnTypeName(JdbcColumn c)
|
83
|
-
{
|
84
|
-
// Redshift does not support TEXT type.
|
85
|
-
switch(c.getSimpleTypeName()) {
|
86
|
-
case "CLOB":
|
87
|
-
return "VARCHAR(65535)";
|
88
|
-
case "TEXT":
|
89
|
-
return "VARCHAR(65535)";
|
90
|
-
case "BLOB":
|
91
|
-
return "BYTEA";
|
92
|
-
default:
|
93
|
-
return super.buildColumnTypeName(c);
|
94
|
-
}
|
95
|
-
}
|
96
|
-
|
97
|
-
public String buildCopySQLBeforeFrom(String tableName, JdbcSchema tableSchema)
|
98
|
-
{
|
99
|
-
StringBuilder sb = new StringBuilder();
|
100
|
-
|
101
|
-
sb.append("COPY ");
|
102
|
-
quoteIdentifierString(sb, tableName);
|
103
|
-
sb.append(" (");
|
104
|
-
for(int i=0; i < tableSchema.getCount(); i++) {
|
105
|
-
if(i != 0) { sb.append(", "); }
|
106
|
-
quoteIdentifierString(sb, tableSchema.getColumnName(i));
|
107
|
-
}
|
108
|
-
sb.append(")");
|
109
|
-
|
110
|
-
return sb.toString();
|
111
|
-
}
|
112
|
-
|
113
|
-
public void runCopy(String sql) throws SQLException
|
114
|
-
{
|
115
|
-
Statement stmt = connection.createStatement();
|
116
|
-
try {
|
117
|
-
stmt.executeUpdate(sql);
|
118
|
-
} finally {
|
119
|
-
stmt.close();
|
120
|
-
}
|
121
|
-
}
|
122
|
-
}
|
1
|
+
package org.embulk.output.redshift;
|
2
|
+
|
3
|
+
import java.sql.Connection;
|
4
|
+
import java.sql.SQLException;
|
5
|
+
import java.sql.Statement;
|
6
|
+
import org.slf4j.Logger;
|
7
|
+
import org.embulk.spi.Exec;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
10
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
11
|
+
|
12
|
+
public class RedshiftOutputConnection
|
13
|
+
extends JdbcOutputConnection
|
14
|
+
{
|
15
|
+
private final Logger logger = Exec.getLogger(RedshiftOutputConnection.class);
|
16
|
+
|
17
|
+
public RedshiftOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
+
throws SQLException
|
19
|
+
{
|
20
|
+
super(connection, schemaName);
|
21
|
+
connection.setAutoCommit(autoCommit);
|
22
|
+
}
|
23
|
+
|
24
|
+
// Redshift does not support DROP TABLE IF EXISTS.
|
25
|
+
// Here runs DROP TABLE and ignores errors.
|
26
|
+
@Override
|
27
|
+
public void dropTableIfExists(String tableName) throws SQLException
|
28
|
+
{
|
29
|
+
Statement stmt = connection.createStatement();
|
30
|
+
try {
|
31
|
+
String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
|
32
|
+
executeUpdate(stmt, sql);
|
33
|
+
commitIfNecessary(connection);
|
34
|
+
} catch (SQLException ex) {
|
35
|
+
// ignore errors.
|
36
|
+
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
37
|
+
SQLException ignored = safeRollback(connection, ex);
|
38
|
+
} finally {
|
39
|
+
stmt.close();
|
40
|
+
}
|
41
|
+
}
|
42
|
+
|
43
|
+
// Redshift does not support DROP TABLE IF EXISTS.
|
44
|
+
// Dropping part runs DROP TABLE and ignores errors.
|
45
|
+
@Override
|
46
|
+
public void replaceTable(String fromTable, JdbcSchema schema, String toTable) throws SQLException
|
47
|
+
{
|
48
|
+
Statement stmt = connection.createStatement();
|
49
|
+
try {
|
50
|
+
try {
|
51
|
+
StringBuilder sb = new StringBuilder();
|
52
|
+
sb.append("DROP TABLE ");
|
53
|
+
quoteIdentifierString(sb, toTable);
|
54
|
+
String sql = sb.toString();
|
55
|
+
executeUpdate(stmt, sql);
|
56
|
+
} catch (SQLException ex) {
|
57
|
+
// ignore errors.
|
58
|
+
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
59
|
+
// rollback or comimt is required to recover failed transaction
|
60
|
+
SQLException ignored = safeRollback(connection, ex);
|
61
|
+
}
|
62
|
+
|
63
|
+
{
|
64
|
+
StringBuilder sb = new StringBuilder();
|
65
|
+
sb.append("ALTER TABLE ");
|
66
|
+
quoteIdentifierString(sb, fromTable);
|
67
|
+
sb.append(" RENAME TO ");
|
68
|
+
quoteIdentifierString(sb, toTable);
|
69
|
+
String sql = sb.toString();
|
70
|
+
executeUpdate(stmt, sql);
|
71
|
+
}
|
72
|
+
|
73
|
+
commitIfNecessary(connection);
|
74
|
+
} catch (SQLException ex) {
|
75
|
+
throw safeRollback(connection, ex);
|
76
|
+
} finally {
|
77
|
+
stmt.close();
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
@Override
|
82
|
+
protected String buildColumnTypeName(JdbcColumn c)
|
83
|
+
{
|
84
|
+
// Redshift does not support TEXT type.
|
85
|
+
switch(c.getSimpleTypeName()) {
|
86
|
+
case "CLOB":
|
87
|
+
return "VARCHAR(65535)";
|
88
|
+
case "TEXT":
|
89
|
+
return "VARCHAR(65535)";
|
90
|
+
case "BLOB":
|
91
|
+
return "BYTEA";
|
92
|
+
default:
|
93
|
+
return super.buildColumnTypeName(c);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
public String buildCopySQLBeforeFrom(String tableName, JdbcSchema tableSchema)
|
98
|
+
{
|
99
|
+
StringBuilder sb = new StringBuilder();
|
100
|
+
|
101
|
+
sb.append("COPY ");
|
102
|
+
quoteIdentifierString(sb, tableName);
|
103
|
+
sb.append(" (");
|
104
|
+
for(int i=0; i < tableSchema.getCount(); i++) {
|
105
|
+
if(i != 0) { sb.append(", "); }
|
106
|
+
quoteIdentifierString(sb, tableSchema.getColumnName(i));
|
107
|
+
}
|
108
|
+
sb.append(")");
|
109
|
+
|
110
|
+
return sb.toString();
|
111
|
+
}
|
112
|
+
|
113
|
+
public void runCopy(String sql) throws SQLException
|
114
|
+
{
|
115
|
+
Statement stmt = connection.createStatement();
|
116
|
+
try {
|
117
|
+
stmt.executeUpdate(sql);
|
118
|
+
} finally {
|
119
|
+
stmt.close();
|
120
|
+
}
|
121
|
+
}
|
122
|
+
}
|
@@ -1,40 +1,40 @@
|
|
1
|
-
package org.embulk.output.redshift;
|
2
|
-
|
3
|
-
import java.util.Properties;
|
4
|
-
import java.sql.Driver;
|
5
|
-
import java.sql.Connection;
|
6
|
-
import java.sql.SQLException;
|
7
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
|
10
|
-
public class RedshiftOutputConnector
|
11
|
-
implements JdbcOutputConnector
|
12
|
-
{
|
13
|
-
private static final Driver driver = new org.postgresql.Driver();
|
14
|
-
|
15
|
-
private final String url;
|
16
|
-
private final Properties properties;
|
17
|
-
private final String schemaName;
|
18
|
-
|
19
|
-
public RedshiftOutputConnector(String url, Properties properties, String schemaName)
|
20
|
-
{
|
21
|
-
this.url = url;
|
22
|
-
this.properties = properties;
|
23
|
-
this.schemaName = schemaName;
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public RedshiftOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
-
{
|
29
|
-
Connection c = driver.connect(url, properties);
|
30
|
-
try {
|
31
|
-
RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName, autoCommit);
|
32
|
-
c = null;
|
33
|
-
return con;
|
34
|
-
} finally {
|
35
|
-
if (c != null) {
|
36
|
-
c.close();
|
37
|
-
}
|
38
|
-
}
|
39
|
-
}
|
40
|
-
}
|
1
|
+
package org.embulk.output.redshift;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.sql.Driver;
|
5
|
+
import java.sql.Connection;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
|
10
|
+
public class RedshiftOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private static final Driver driver = new org.postgresql.Driver();
|
14
|
+
|
15
|
+
private final String url;
|
16
|
+
private final Properties properties;
|
17
|
+
private final String schemaName;
|
18
|
+
|
19
|
+
public RedshiftOutputConnector(String url, Properties properties, String schemaName)
|
20
|
+
{
|
21
|
+
this.url = url;
|
22
|
+
this.properties = properties;
|
23
|
+
this.schemaName = schemaName;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public RedshiftOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
+
{
|
29
|
+
Connection c = driver.connect(url, properties);
|
30
|
+
try {
|
31
|
+
RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName, autoCommit);
|
32
|
+
c = null;
|
33
|
+
return con;
|
34
|
+
} finally {
|
35
|
+
if (c != null) {
|
36
|
+
c.close();
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-08 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -24,23 +24,18 @@ files:
|
|
24
24
|
- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
|
25
25
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
|
26
26
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
|
27
|
-
- classpath/aws-java-sdk-core-1.
|
28
|
-
- classpath/aws-java-sdk-kms-1.
|
29
|
-
- classpath/aws-java-sdk-s3-1.
|
30
|
-
- classpath/aws-java-sdk-sts-1.
|
27
|
+
- classpath/aws-java-sdk-core-1.10.33.jar
|
28
|
+
- classpath/aws-java-sdk-kms-1.10.33.jar
|
29
|
+
- classpath/aws-java-sdk-s3-1.10.33.jar
|
30
|
+
- classpath/aws-java-sdk-sts-1.10.33.jar
|
31
31
|
- classpath/commons-codec-1.6.jar
|
32
32
|
- classpath/commons-logging-1.1.3.jar
|
33
|
-
- classpath/embulk-output-jdbc-0.4.
|
34
|
-
- classpath/embulk-output-postgresql-0.4.
|
35
|
-
- classpath/embulk-output-redshift-0.4.
|
36
|
-
- classpath/httpclient-4.3.
|
37
|
-
- classpath/httpcore-4.3.
|
38
|
-
- classpath/
|
39
|
-
- classpath/jna-platform-4.1.0.jar
|
40
|
-
- classpath/joda-time-2.8.1.jar
|
41
|
-
- classpath/postgresql-9.4-1200-jdbc41.jar
|
42
|
-
- classpath/slf4j-simple-1.7.7.jar
|
43
|
-
- classpath/waffle-jna-1.7.jar
|
33
|
+
- classpath/embulk-output-jdbc-0.4.2.jar
|
34
|
+
- classpath/embulk-output-postgresql-0.4.2.jar
|
35
|
+
- classpath/embulk-output-redshift-0.4.2.jar
|
36
|
+
- classpath/httpclient-4.3.6.jar
|
37
|
+
- classpath/httpcore-4.3.3.jar
|
38
|
+
- classpath/postgresql-9.4-1205-jdbc41.jar
|
44
39
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
45
40
|
licenses:
|
46
41
|
- Apache 2.0
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/classpath/jna-4.1.0.jar
DELETED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|