embulk-output-redshift 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/classpath/embulk-output-jdbc-0.2.0.jar +0 -0
- data/classpath/{embulk-output-postgresql-0.1.2.jar → embulk-output-postgresql-0.2.0.jar} +0 -0
- data/classpath/{embulk-output-redshift-0.1.2.jar → embulk-output-redshift-0.2.0.jar} +0 -0
- data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java +32 -11
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java +5 -6
- metadata +5 -5
- data/classpath/embulk-output-jdbc-0.1.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e3ce6b0831abd53dfa802eb0ac5744b7bf33c90c
|
4
|
+
data.tar.gz: c123cce8fff3f0fcc3088bf68267de9f868f14bb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b7a51d83ebe03ac82abf75188a74190dc6466839f6ed9ec74474d9f48d620f922f33d2acc33bfc0625a94dcd52327a1fc5326a459cc3fa914fdd2d514210411b
|
7
|
+
data.tar.gz: 9fbd5a53e11b5e15303cb139499acac8585b22be265bf86ec4385885395fe73a08909060d566286f77ef9b705cb7e0b5e20e74b2cbdbabbac3fd0d2a0ce79524
|
data/README.md
CHANGED
@@ -17,8 +17,8 @@ Redshift output plugins for Embulk loads records to Redshift.
|
|
17
17
|
- **user**: database login user name (string, required)
|
18
18
|
- **password**: database login password (string, default: "")
|
19
19
|
- **database**: destination database name (string, required)
|
20
|
-
- **schema**: destination name (string, default: "public")
|
21
|
-
- **table**: destination name (string, required)
|
20
|
+
- **schema**: destination schema name (string, default: "public")
|
21
|
+
- **table**: destination table name (string, required)
|
22
22
|
- **mode**: "replace" or "insert" (string, required)
|
23
23
|
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
24
24
|
- **options**: extra connection properties (hash, default: {})
|
Binary file
|
Binary file
|
Binary file
|
@@ -8,6 +8,7 @@ import com.amazonaws.auth.AWSCredentials;
|
|
8
8
|
import com.amazonaws.auth.BasicAWSCredentials;
|
9
9
|
import org.embulk.spi.Exec;
|
10
10
|
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
11
12
|
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
12
13
|
import org.embulk.output.jdbc.BatchInsert;
|
13
14
|
import org.embulk.output.redshift.RedshiftOutputConnector;
|
@@ -16,13 +17,31 @@ import org.embulk.output.redshift.RedshiftCopyBatchInsert;
|
|
16
17
|
public class RedshiftOutputPlugin
|
17
18
|
extends AbstractJdbcOutputPlugin
|
18
19
|
{
|
19
|
-
private static final String DEFAULT_SCHEMA = "public";
|
20
|
-
private static final int DEFAULT_PORT = 5439;
|
21
|
-
|
22
20
|
private final Logger logger = Exec.getLogger(RedshiftOutputPlugin.class);
|
23
21
|
|
24
22
|
public interface RedshiftPluginTask extends PluginTask
|
25
23
|
{
|
24
|
+
@Config("host")
|
25
|
+
public String getHost();
|
26
|
+
|
27
|
+
@Config("port")
|
28
|
+
@ConfigDefault("5439")
|
29
|
+
public int getPort();
|
30
|
+
|
31
|
+
@Config("user")
|
32
|
+
public String getUser();
|
33
|
+
|
34
|
+
@Config("password")
|
35
|
+
@ConfigDefault("\"\"")
|
36
|
+
public String getPassword();
|
37
|
+
|
38
|
+
@Config("database")
|
39
|
+
public String getDatabase();
|
40
|
+
|
41
|
+
@Config("schema")
|
42
|
+
@ConfigDefault("\"public\"")
|
43
|
+
public String getSchema();
|
44
|
+
|
26
45
|
@Config("access_key_id")
|
27
46
|
public String getAccessKeyId();
|
28
47
|
|
@@ -45,12 +64,14 @@ public class RedshiftOutputPlugin
|
|
45
64
|
@Override
|
46
65
|
protected RedshiftOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
47
66
|
{
|
67
|
+
RedshiftPluginTask t = (RedshiftPluginTask) task;
|
68
|
+
|
48
69
|
String url = String.format("jdbc:postgresql://%s:%d/%s",
|
49
|
-
|
70
|
+
t.getHost(), t.getPort(), t.getDatabase());
|
50
71
|
|
51
72
|
Properties props = new Properties();
|
52
|
-
props.setProperty("user",
|
53
|
-
props.setProperty("password",
|
73
|
+
props.setProperty("user", t.getUser());
|
74
|
+
props.setProperty("password", t.getPassword());
|
54
75
|
props.setProperty("loginTimeout", "300"); // seconds
|
55
76
|
props.setProperty("socketTimeout", "1800"); // seconds
|
56
77
|
|
@@ -75,18 +96,18 @@ public class RedshiftOutputPlugin
|
|
75
96
|
props.setProperty("socketTimeout", "28800"); // seconds
|
76
97
|
}
|
77
98
|
|
78
|
-
props.putAll(
|
99
|
+
props.putAll(t.getOptions());
|
79
100
|
|
80
|
-
return new RedshiftOutputConnector(url, props,
|
101
|
+
return new RedshiftOutputConnector(url, props, t.getSchema());
|
81
102
|
}
|
82
103
|
|
83
104
|
@Override
|
84
105
|
protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
|
85
106
|
{
|
86
|
-
RedshiftPluginTask
|
107
|
+
RedshiftPluginTask t = (RedshiftPluginTask) task;
|
87
108
|
AWSCredentials creds = new BasicAWSCredentials(
|
88
|
-
|
109
|
+
t.getAccessKeyId(), t.getSecretAccessKey());
|
89
110
|
return new RedshiftCopyBatchInsert(getConnector(task, true),
|
90
|
-
creds,
|
111
|
+
creds, t.getS3Bucket(), t.getIamUserName());
|
91
112
|
}
|
92
113
|
}
|
@@ -30,11 +30,11 @@ public class RedshiftOutputConnection
|
|
30
30
|
try {
|
31
31
|
String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
|
32
32
|
executeUpdate(stmt, sql);
|
33
|
-
connection
|
33
|
+
commitIfNecessary(connection);
|
34
34
|
} catch (SQLException ex) {
|
35
35
|
// ignore errors.
|
36
36
|
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
37
|
-
connection
|
37
|
+
SQLException ignored = safeRollback(connection, ex);
|
38
38
|
} finally {
|
39
39
|
stmt.close();
|
40
40
|
}
|
@@ -57,7 +57,7 @@ public class RedshiftOutputConnection
|
|
57
57
|
// ignore errors.
|
58
58
|
// TODO here should ignore only 'table "XXX" does not exist' errors.
|
59
59
|
// rollback or comimt is required to recover failed transaction
|
60
|
-
connection
|
60
|
+
SQLException ignored = safeRollback(connection, ex);
|
61
61
|
}
|
62
62
|
|
63
63
|
{
|
@@ -70,10 +70,9 @@ public class RedshiftOutputConnection
|
|
70
70
|
executeUpdate(stmt, sql);
|
71
71
|
}
|
72
72
|
|
73
|
-
connection
|
73
|
+
commitIfNecessary(connection);
|
74
74
|
} catch (SQLException ex) {
|
75
|
-
connection
|
76
|
-
throw ex;
|
75
|
+
throw safeRollback(connection, ex);
|
77
76
|
} finally {
|
78
77
|
stmt.close();
|
79
78
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-28 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -30,9 +30,9 @@ files:
|
|
30
30
|
- classpath/aws-java-sdk-sts-1.9.17.jar
|
31
31
|
- classpath/commons-codec-1.6.jar
|
32
32
|
- classpath/commons-logging-1.1.3.jar
|
33
|
-
- classpath/embulk-output-jdbc-0.
|
34
|
-
- classpath/embulk-output-postgresql-0.
|
35
|
-
- classpath/embulk-output-redshift-0.
|
33
|
+
- classpath/embulk-output-jdbc-0.2.0.jar
|
34
|
+
- classpath/embulk-output-postgresql-0.2.0.jar
|
35
|
+
- classpath/embulk-output-redshift-0.2.0.jar
|
36
36
|
- classpath/httpclient-4.3.4.jar
|
37
37
|
- classpath/httpcore-4.3.2.jar
|
38
38
|
- classpath/jna-4.1.0.jar
|
Binary file
|