embulk-output-redshift 0.4.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +104 -104
- data/build.gradle +9 -9
- data/classpath/{aws-java-sdk-core-1.9.17.jar → aws-java-sdk-core-1.10.33.jar} +0 -0
- data/classpath/aws-java-sdk-kms-1.10.33.jar +0 -0
- data/classpath/aws-java-sdk-s3-1.10.33.jar +0 -0
- data/classpath/aws-java-sdk-sts-1.10.33.jar +0 -0
- data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
- data/classpath/embulk-output-postgresql-0.4.2.jar +0 -0
- data/classpath/{embulk-output-redshift-0.4.1.jar → embulk-output-redshift-0.4.2.jar} +0 -0
- data/classpath/{httpclient-4.3.4.jar → httpclient-4.3.6.jar} +0 -0
- data/classpath/{httpcore-4.3.2.jar → httpcore-4.3.3.jar} +0 -0
- data/classpath/postgresql-9.4-1205-jdbc41.jar +0 -0
- data/lib/embulk/output/redshift.rb +3 -3
- data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java +151 -151
- data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java +218 -218
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java +122 -122
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java +40 -40
- metadata +12 -17
- data/classpath/aws-java-sdk-kms-1.9.17.jar +0 -0
- data/classpath/aws-java-sdk-s3-1.9.17.jar +0 -0
- data/classpath/aws-java-sdk-sts-1.9.17.jar +0 -0
- data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
- data/classpath/embulk-output-postgresql-0.4.1.jar +0 -0
- data/classpath/jna-4.1.0.jar +0 -0
- data/classpath/jna-platform-4.1.0.jar +0 -0
- data/classpath/joda-time-2.8.1.jar +0 -0
- data/classpath/postgresql-9.4-1200-jdbc41.jar +0 -0
- data/classpath/slf4j-simple-1.7.7.jar +0 -0
- data/classpath/waffle-jna-1.7.jar +0 -0
@@ -1,122 +1,122 @@
|
|
1
|
-
package org.embulk.output.redshift;
|
2
|
-
|
3
|
-
import java.sql.Connection;
|
4
|
-
import java.sql.SQLException;
|
5
|
-
import java.sql.Statement;
|
6
|
-
import org.slf4j.Logger;
|
7
|
-
import org.embulk.spi.Exec;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
10
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
11
|
-
|
12
|
-
public class RedshiftOutputConnection
|
13
|
-
extends JdbcOutputConnection
|
14
|
-
{
|
15
|
-
private final Logger logger = Exec.getLogger(RedshiftOutputConnection.class);
|
16
|
-
|
17
|
-
public RedshiftOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
-
throws SQLException
|
19
|
-
{
|
20
|
-
super(connection, schemaName);
|
21
|
-
connection.setAutoCommit(autoCommit);
|
22
|
-
}
|
23
|
-
|
24
|
-
// Redshift does not support DROP TABLE IF EXISTS.
|
25
|
-
// Here runs DROP TABLE and ignores errors.
|
26
|
-
@Override
|
27
|
-
public void dropTableIfExists(String tableName) throws SQLException
|
28
|
-
{
|
29
|
-
Statement stmt = connection.createStatement();
|
30
|
-
try {
|
31
|
-
String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
|
32
|
-
executeUpdate(stmt, sql);
|
33
|
-
commitIfNecessary(connection);
|
34
|
-
} catch (SQLException ex) {
|
35
|
-
// ignore errors.
|
36
|
-
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
37
|
-
SQLException ignored = safeRollback(connection, ex);
|
38
|
-
} finally {
|
39
|
-
stmt.close();
|
40
|
-
}
|
41
|
-
}
|
42
|
-
|
43
|
-
// Redshift does not support DROP TABLE IF EXISTS.
|
44
|
-
// Dropping part runs DROP TABLE and ignores errors.
|
45
|
-
@Override
|
46
|
-
public void replaceTable(String fromTable, JdbcSchema schema, String toTable) throws SQLException
|
47
|
-
{
|
48
|
-
Statement stmt = connection.createStatement();
|
49
|
-
try {
|
50
|
-
try {
|
51
|
-
StringBuilder sb = new StringBuilder();
|
52
|
-
sb.append("DROP TABLE ");
|
53
|
-
quoteIdentifierString(sb, toTable);
|
54
|
-
String sql = sb.toString();
|
55
|
-
executeUpdate(stmt, sql);
|
56
|
-
} catch (SQLException ex) {
|
57
|
-
// ignore errors.
|
58
|
-
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
59
|
-
// rollback or comimt is required to recover failed transaction
|
60
|
-
SQLException ignored = safeRollback(connection, ex);
|
61
|
-
}
|
62
|
-
|
63
|
-
{
|
64
|
-
StringBuilder sb = new StringBuilder();
|
65
|
-
sb.append("ALTER TABLE ");
|
66
|
-
quoteIdentifierString(sb, fromTable);
|
67
|
-
sb.append(" RENAME TO ");
|
68
|
-
quoteIdentifierString(sb, toTable);
|
69
|
-
String sql = sb.toString();
|
70
|
-
executeUpdate(stmt, sql);
|
71
|
-
}
|
72
|
-
|
73
|
-
commitIfNecessary(connection);
|
74
|
-
} catch (SQLException ex) {
|
75
|
-
throw safeRollback(connection, ex);
|
76
|
-
} finally {
|
77
|
-
stmt.close();
|
78
|
-
}
|
79
|
-
}
|
80
|
-
|
81
|
-
@Override
|
82
|
-
protected String buildColumnTypeName(JdbcColumn c)
|
83
|
-
{
|
84
|
-
// Redshift does not support TEXT type.
|
85
|
-
switch(c.getSimpleTypeName()) {
|
86
|
-
case "CLOB":
|
87
|
-
return "VARCHAR(65535)";
|
88
|
-
case "TEXT":
|
89
|
-
return "VARCHAR(65535)";
|
90
|
-
case "BLOB":
|
91
|
-
return "BYTEA";
|
92
|
-
default:
|
93
|
-
return super.buildColumnTypeName(c);
|
94
|
-
}
|
95
|
-
}
|
96
|
-
|
97
|
-
public String buildCopySQLBeforeFrom(String tableName, JdbcSchema tableSchema)
|
98
|
-
{
|
99
|
-
StringBuilder sb = new StringBuilder();
|
100
|
-
|
101
|
-
sb.append("COPY ");
|
102
|
-
quoteIdentifierString(sb, tableName);
|
103
|
-
sb.append(" (");
|
104
|
-
for(int i=0; i < tableSchema.getCount(); i++) {
|
105
|
-
if(i != 0) { sb.append(", "); }
|
106
|
-
quoteIdentifierString(sb, tableSchema.getColumnName(i));
|
107
|
-
}
|
108
|
-
sb.append(")");
|
109
|
-
|
110
|
-
return sb.toString();
|
111
|
-
}
|
112
|
-
|
113
|
-
public void runCopy(String sql) throws SQLException
|
114
|
-
{
|
115
|
-
Statement stmt = connection.createStatement();
|
116
|
-
try {
|
117
|
-
stmt.executeUpdate(sql);
|
118
|
-
} finally {
|
119
|
-
stmt.close();
|
120
|
-
}
|
121
|
-
}
|
122
|
-
}
|
1
|
+
package org.embulk.output.redshift;
|
2
|
+
|
3
|
+
import java.sql.Connection;
|
4
|
+
import java.sql.SQLException;
|
5
|
+
import java.sql.Statement;
|
6
|
+
import org.slf4j.Logger;
|
7
|
+
import org.embulk.spi.Exec;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
10
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
11
|
+
|
12
|
+
public class RedshiftOutputConnection
|
13
|
+
extends JdbcOutputConnection
|
14
|
+
{
|
15
|
+
private final Logger logger = Exec.getLogger(RedshiftOutputConnection.class);
|
16
|
+
|
17
|
+
public RedshiftOutputConnection(Connection connection, String schemaName, boolean autoCommit)
|
18
|
+
throws SQLException
|
19
|
+
{
|
20
|
+
super(connection, schemaName);
|
21
|
+
connection.setAutoCommit(autoCommit);
|
22
|
+
}
|
23
|
+
|
24
|
+
// Redshift does not support DROP TABLE IF EXISTS.
|
25
|
+
// Here runs DROP TABLE and ignores errors.
|
26
|
+
@Override
|
27
|
+
public void dropTableIfExists(String tableName) throws SQLException
|
28
|
+
{
|
29
|
+
Statement stmt = connection.createStatement();
|
30
|
+
try {
|
31
|
+
String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
|
32
|
+
executeUpdate(stmt, sql);
|
33
|
+
commitIfNecessary(connection);
|
34
|
+
} catch (SQLException ex) {
|
35
|
+
// ignore errors.
|
36
|
+
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
37
|
+
SQLException ignored = safeRollback(connection, ex);
|
38
|
+
} finally {
|
39
|
+
stmt.close();
|
40
|
+
}
|
41
|
+
}
|
42
|
+
|
43
|
+
// Redshift does not support DROP TABLE IF EXISTS.
|
44
|
+
// Dropping part runs DROP TABLE and ignores errors.
|
45
|
+
@Override
|
46
|
+
public void replaceTable(String fromTable, JdbcSchema schema, String toTable) throws SQLException
|
47
|
+
{
|
48
|
+
Statement stmt = connection.createStatement();
|
49
|
+
try {
|
50
|
+
try {
|
51
|
+
StringBuilder sb = new StringBuilder();
|
52
|
+
sb.append("DROP TABLE ");
|
53
|
+
quoteIdentifierString(sb, toTable);
|
54
|
+
String sql = sb.toString();
|
55
|
+
executeUpdate(stmt, sql);
|
56
|
+
} catch (SQLException ex) {
|
57
|
+
// ignore errors.
|
58
|
+
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
59
|
+
// rollback or comimt is required to recover failed transaction
|
60
|
+
SQLException ignored = safeRollback(connection, ex);
|
61
|
+
}
|
62
|
+
|
63
|
+
{
|
64
|
+
StringBuilder sb = new StringBuilder();
|
65
|
+
sb.append("ALTER TABLE ");
|
66
|
+
quoteIdentifierString(sb, fromTable);
|
67
|
+
sb.append(" RENAME TO ");
|
68
|
+
quoteIdentifierString(sb, toTable);
|
69
|
+
String sql = sb.toString();
|
70
|
+
executeUpdate(stmt, sql);
|
71
|
+
}
|
72
|
+
|
73
|
+
commitIfNecessary(connection);
|
74
|
+
} catch (SQLException ex) {
|
75
|
+
throw safeRollback(connection, ex);
|
76
|
+
} finally {
|
77
|
+
stmt.close();
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
@Override
|
82
|
+
protected String buildColumnTypeName(JdbcColumn c)
|
83
|
+
{
|
84
|
+
// Redshift does not support TEXT type.
|
85
|
+
switch(c.getSimpleTypeName()) {
|
86
|
+
case "CLOB":
|
87
|
+
return "VARCHAR(65535)";
|
88
|
+
case "TEXT":
|
89
|
+
return "VARCHAR(65535)";
|
90
|
+
case "BLOB":
|
91
|
+
return "BYTEA";
|
92
|
+
default:
|
93
|
+
return super.buildColumnTypeName(c);
|
94
|
+
}
|
95
|
+
}
|
96
|
+
|
97
|
+
public String buildCopySQLBeforeFrom(String tableName, JdbcSchema tableSchema)
|
98
|
+
{
|
99
|
+
StringBuilder sb = new StringBuilder();
|
100
|
+
|
101
|
+
sb.append("COPY ");
|
102
|
+
quoteIdentifierString(sb, tableName);
|
103
|
+
sb.append(" (");
|
104
|
+
for(int i=0; i < tableSchema.getCount(); i++) {
|
105
|
+
if(i != 0) { sb.append(", "); }
|
106
|
+
quoteIdentifierString(sb, tableSchema.getColumnName(i));
|
107
|
+
}
|
108
|
+
sb.append(")");
|
109
|
+
|
110
|
+
return sb.toString();
|
111
|
+
}
|
112
|
+
|
113
|
+
public void runCopy(String sql) throws SQLException
|
114
|
+
{
|
115
|
+
Statement stmt = connection.createStatement();
|
116
|
+
try {
|
117
|
+
stmt.executeUpdate(sql);
|
118
|
+
} finally {
|
119
|
+
stmt.close();
|
120
|
+
}
|
121
|
+
}
|
122
|
+
}
|
@@ -1,40 +1,40 @@
|
|
1
|
-
package org.embulk.output.redshift;
|
2
|
-
|
3
|
-
import java.util.Properties;
|
4
|
-
import java.sql.Driver;
|
5
|
-
import java.sql.Connection;
|
6
|
-
import java.sql.SQLException;
|
7
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
-
|
10
|
-
public class RedshiftOutputConnector
|
11
|
-
implements JdbcOutputConnector
|
12
|
-
{
|
13
|
-
private static final Driver driver = new org.postgresql.Driver();
|
14
|
-
|
15
|
-
private final String url;
|
16
|
-
private final Properties properties;
|
17
|
-
private final String schemaName;
|
18
|
-
|
19
|
-
public RedshiftOutputConnector(String url, Properties properties, String schemaName)
|
20
|
-
{
|
21
|
-
this.url = url;
|
22
|
-
this.properties = properties;
|
23
|
-
this.schemaName = schemaName;
|
24
|
-
}
|
25
|
-
|
26
|
-
@Override
|
27
|
-
public RedshiftOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
-
{
|
29
|
-
Connection c = driver.connect(url, properties);
|
30
|
-
try {
|
31
|
-
RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName, autoCommit);
|
32
|
-
c = null;
|
33
|
-
return con;
|
34
|
-
} finally {
|
35
|
-
if (c != null) {
|
36
|
-
c.close();
|
37
|
-
}
|
38
|
-
}
|
39
|
-
}
|
40
|
-
}
|
1
|
+
package org.embulk.output.redshift;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.sql.Driver;
|
5
|
+
import java.sql.Connection;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
9
|
+
|
10
|
+
public class RedshiftOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private static final Driver driver = new org.postgresql.Driver();
|
14
|
+
|
15
|
+
private final String url;
|
16
|
+
private final Properties properties;
|
17
|
+
private final String schemaName;
|
18
|
+
|
19
|
+
public RedshiftOutputConnector(String url, Properties properties, String schemaName)
|
20
|
+
{
|
21
|
+
this.url = url;
|
22
|
+
this.properties = properties;
|
23
|
+
this.schemaName = schemaName;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public RedshiftOutputConnection connect(boolean autoCommit) throws SQLException
|
28
|
+
{
|
29
|
+
Connection c = driver.connect(url, properties);
|
30
|
+
try {
|
31
|
+
RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName, autoCommit);
|
32
|
+
c = null;
|
33
|
+
return con;
|
34
|
+
} finally {
|
35
|
+
if (c != null) {
|
36
|
+
c.close();
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
40
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-08 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -24,23 +24,18 @@ files:
|
|
24
24
|
- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
|
25
25
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
|
26
26
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
|
27
|
-
- classpath/aws-java-sdk-core-1.
|
28
|
-
- classpath/aws-java-sdk-kms-1.
|
29
|
-
- classpath/aws-java-sdk-s3-1.
|
30
|
-
- classpath/aws-java-sdk-sts-1.
|
27
|
+
- classpath/aws-java-sdk-core-1.10.33.jar
|
28
|
+
- classpath/aws-java-sdk-kms-1.10.33.jar
|
29
|
+
- classpath/aws-java-sdk-s3-1.10.33.jar
|
30
|
+
- classpath/aws-java-sdk-sts-1.10.33.jar
|
31
31
|
- classpath/commons-codec-1.6.jar
|
32
32
|
- classpath/commons-logging-1.1.3.jar
|
33
|
-
- classpath/embulk-output-jdbc-0.4.
|
34
|
-
- classpath/embulk-output-postgresql-0.4.
|
35
|
-
- classpath/embulk-output-redshift-0.4.
|
36
|
-
- classpath/httpclient-4.3.
|
37
|
-
- classpath/httpcore-4.3.
|
38
|
-
- classpath/
|
39
|
-
- classpath/jna-platform-4.1.0.jar
|
40
|
-
- classpath/joda-time-2.8.1.jar
|
41
|
-
- classpath/postgresql-9.4-1200-jdbc41.jar
|
42
|
-
- classpath/slf4j-simple-1.7.7.jar
|
43
|
-
- classpath/waffle-jna-1.7.jar
|
33
|
+
- classpath/embulk-output-jdbc-0.4.2.jar
|
34
|
+
- classpath/embulk-output-postgresql-0.4.2.jar
|
35
|
+
- classpath/embulk-output-redshift-0.4.2.jar
|
36
|
+
- classpath/httpclient-4.3.6.jar
|
37
|
+
- classpath/httpcore-4.3.3.jar
|
38
|
+
- classpath/postgresql-9.4-1205-jdbc41.jar
|
44
39
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
45
40
|
licenses:
|
46
41
|
- Apache 2.0
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
data/classpath/jna-4.1.0.jar
DELETED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|