embulk-output-redshift 0.6.1 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +9 -4
- data/classpath/embulk-output-jdbc-0.6.2.jar +0 -0
- data/classpath/embulk-output-postgresql-0.6.2.jar +0 -0
- data/classpath/embulk-output-redshift-0.6.2.jar +0 -0
- data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java +21 -12
- data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java +5 -1
- data/src/main/java/org/embulk/output/redshift/Ssl.java +37 -0
- metadata +6 -5
- data/classpath/embulk-output-jdbc-0.6.1.jar +0 -0
- data/classpath/embulk-output-postgresql-0.6.1.jar +0 -0
- data/classpath/embulk-output-redshift-0.6.1.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6b2e8b89ca25df20d98cf01a706f0b9667d291c7
|
4
|
+
data.tar.gz: 58536c435cfce02d53f966305515c703c782797f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 23d34857b174f1782c2c487d4c631a22d0aa5e0c6ff809e29302ccd8c1c0d51cf30012428453fdc43a47638b2681ba338ffb3f0c8786583db71e039dedffb84e
|
7
|
+
data.tar.gz: 0cd7a1c77dae1c65381197da11468c63a35db75972a81237e35fc30f1f894b36b486c75b2680bb4e63a9d710f07923e96cb022ca5e54a2b8479230ea82e6118f
|
data/README.md
CHANGED
@@ -5,14 +5,15 @@ Redshift output plugins for Embulk loads records to Redshift.
|
|
5
5
|
## Overview
|
6
6
|
|
7
7
|
* **Plugin type**: output
|
8
|
-
* **Load all or nothing**:
|
9
|
-
* **Resume supported**:
|
8
|
+
* **Load all or nothing**: depends on the mode. see below.
|
9
|
+
* **Resume supported**: depends on the mode. see below.
|
10
10
|
|
11
11
|
## Configuration
|
12
12
|
|
13
13
|
- **host**: database host name (string, required)
|
14
14
|
- **port**: database port number (integer, default: 5439)
|
15
15
|
- **user**: database login user name (string, required)
|
16
|
+
- **ssl**: use SSL to connect to the database (string, default: "disable". "enable" uses SSL without server-side validation and "verify" checks the certificate. For compatibility reasons, "true" behaves as "enable" and "false" behaves as "disable".)
|
16
17
|
- **password**: database login password (string, default: "")
|
17
18
|
- **database**: destination database name (string, required)
|
18
19
|
- **schema**: destination schema name (string, default: "public")
|
@@ -23,6 +24,9 @@ Redshift output plugins for Embulk loads records to Redshift.
|
|
23
24
|
- **s3_bucket**: S3 bucket name for temporary files
|
24
25
|
- **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
|
25
26
|
- **options**: extra connection properties (hash, default: {})
|
27
|
+
- **retry_limit** max retry count for database operations (integer, default: 12)
|
28
|
+
- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
|
29
|
+
- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
|
26
30
|
- **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)
|
27
31
|
- **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode)
|
28
32
|
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
@@ -100,6 +104,7 @@ out:
|
|
100
104
|
type: redshift
|
101
105
|
host: myinstance.us-west-2.redshift.amazonaws.com
|
102
106
|
user: pg
|
107
|
+
ssl: enable
|
103
108
|
password: ""
|
104
109
|
database: my_database
|
105
110
|
table: my_table
|
@@ -120,10 +125,10 @@ out:
|
|
120
125
|
### Build
|
121
126
|
|
122
127
|
```
|
123
|
-
$
|
128
|
+
$ ../gradlew gem
|
124
129
|
```
|
125
130
|
|
126
131
|
### Security
|
127
132
|
This plugin requires AWS access credentials so that it may write temporary files to S3. There are two security options, Standard and Federated.
|
128
133
|
To use Standard security, give **aws_key_id** and **secret_access_key**. To use Federated mode, also give the **iam_user_name** field.
|
129
|
-
Federated mode really means temporary credentials, so that a man-in-the-middle attack will see AWS credentials that are only valid for 1 calendar day after the transaction.
|
134
|
+
Federated mode really means temporary credentials, so that a man-in-the-middle attack will see AWS credentials that are only valid for 1 calendar day after the transaction.
|
Binary file
|
Binary file
|
Binary file
|
@@ -4,6 +4,8 @@ import java.util.List;
|
|
4
4
|
import java.util.Properties;
|
5
5
|
import java.io.IOException;
|
6
6
|
import java.sql.SQLException;
|
7
|
+
|
8
|
+
import org.embulk.output.jdbc.MergeConfig;
|
7
9
|
import org.slf4j.Logger;
|
8
10
|
import com.google.common.base.Optional;
|
9
11
|
import com.google.common.collect.ImmutableSet;
|
@@ -17,6 +19,7 @@ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
|
17
19
|
import org.embulk.output.jdbc.BatchInsert;
|
18
20
|
import org.embulk.output.redshift.RedshiftOutputConnector;
|
19
21
|
import org.embulk.output.redshift.RedshiftCopyBatchInsert;
|
22
|
+
import org.embulk.output.redshift.Ssl;
|
20
23
|
|
21
24
|
public class RedshiftOutputPlugin
|
22
25
|
extends AbstractJdbcOutputPlugin
|
@@ -62,6 +65,10 @@ public class RedshiftOutputPlugin
|
|
62
65
|
@Config("s3_key_prefix")
|
63
66
|
@ConfigDefault("\"\"")
|
64
67
|
public String getS3KeyPrefix();
|
68
|
+
|
69
|
+
@Config("ssl")
|
70
|
+
@ConfigDefault("disable")
|
71
|
+
public Ssl getSsl();
|
65
72
|
}
|
66
73
|
|
67
74
|
@Override
|
@@ -95,16 +102,18 @@ public class RedshiftOutputPlugin
|
|
95
102
|
// Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
|
96
103
|
props.setProperty("tcpKeepAlive", "true");
|
97
104
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
105
|
+
switch (t.getSsl()) {
|
106
|
+
case DISABLE:
|
107
|
+
break;
|
108
|
+
case ENABLE:
|
109
|
+
// See http://docs.aws.amazon.com/redshift/latest/mgmt/connecting-ssl-support.html
|
110
|
+
props.setProperty("ssl", "true");
|
111
|
+
props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
|
112
|
+
break;
|
113
|
+
case VERIFY:
|
114
|
+
props.setProperty("ssl", "true");
|
115
|
+
break;
|
116
|
+
}
|
108
117
|
|
109
118
|
if (!retryableMetadataOperation) {
|
110
119
|
// non-retryable batch operation uses longer timeout
|
@@ -140,9 +149,9 @@ public class RedshiftOutputPlugin
|
|
140
149
|
}
|
141
150
|
|
142
151
|
@Override
|
143
|
-
protected BatchInsert newBatchInsert(PluginTask task, Optional<
|
152
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
|
144
153
|
{
|
145
|
-
if (
|
154
|
+
if (mergeConfig.isPresent()) {
|
146
155
|
throw new UnsupportedOperationException("Redshift output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
|
147
156
|
}
|
148
157
|
RedshiftPluginTask t = (RedshiftPluginTask) task;
|
@@ -4,6 +4,9 @@ import java.util.List;
|
|
4
4
|
import java.sql.Connection;
|
5
5
|
import java.sql.SQLException;
|
6
6
|
import java.sql.Statement;
|
7
|
+
|
8
|
+
import com.google.common.base.Optional;
|
9
|
+
import org.embulk.output.jdbc.MergeConfig;
|
7
10
|
import org.slf4j.Logger;
|
8
11
|
import org.embulk.spi.Exec;
|
9
12
|
import org.embulk.output.jdbc.JdbcOutputConnection;
|
@@ -122,7 +125,7 @@ public class RedshiftOutputConnection
|
|
122
125
|
}
|
123
126
|
|
124
127
|
@Override
|
125
|
-
protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable,
|
128
|
+
protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
|
126
129
|
{
|
127
130
|
StringBuilder sb = new StringBuilder();
|
128
131
|
|
@@ -137,6 +140,7 @@ public class RedshiftOutputConnection
|
|
137
140
|
quoteIdentifierString(sb, fromTables.get(i));
|
138
141
|
}
|
139
142
|
sb.append(") S WHERE (");
|
143
|
+
List<String> mergeKeys = mergeConfig.getMergeKeys();
|
140
144
|
for(int i=0; i < mergeKeys.size(); i++) {
|
141
145
|
if (i != 0) { sb.append(" AND "); }
|
142
146
|
sb.append("S.");
|
@@ -0,0 +1,37 @@
|
|
1
|
+
package org.embulk.output.redshift;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
5
|
+
|
6
|
+
import org.embulk.config.ConfigException;
|
7
|
+
|
8
|
+
public enum Ssl
|
9
|
+
{
|
10
|
+
ENABLE,
|
11
|
+
DISABLE,
|
12
|
+
VERIFY;
|
13
|
+
|
14
|
+
@JsonValue
|
15
|
+
@Override
|
16
|
+
public String toString()
|
17
|
+
{
|
18
|
+
return this.name().toLowerCase();
|
19
|
+
}
|
20
|
+
|
21
|
+
@JsonCreator
|
22
|
+
public static Ssl fromString(String value)
|
23
|
+
{
|
24
|
+
switch(value) {
|
25
|
+
case "enable":
|
26
|
+
case "true":
|
27
|
+
return ENABLE;
|
28
|
+
case "disable":
|
29
|
+
case "false":
|
30
|
+
return DISABLE;
|
31
|
+
case "verify":
|
32
|
+
return VERIFY;
|
33
|
+
default:
|
34
|
+
throw new ConfigException(String.format("Unknown SSL value '%s'. Supported values are enable, true, disable, false or verify.", value));
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -24,15 +24,16 @@ files:
|
|
24
24
|
- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
|
25
25
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
|
26
26
|
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
|
27
|
+
- src/main/java/org/embulk/output/redshift/Ssl.java
|
27
28
|
- classpath/aws-java-sdk-core-1.10.33.jar
|
28
29
|
- classpath/aws-java-sdk-kms-1.10.33.jar
|
29
30
|
- classpath/aws-java-sdk-s3-1.10.33.jar
|
30
31
|
- classpath/aws-java-sdk-sts-1.10.33.jar
|
31
32
|
- classpath/commons-codec-1.6.jar
|
32
33
|
- classpath/commons-logging-1.1.3.jar
|
33
|
-
- classpath/embulk-output-jdbc-0.6.
|
34
|
-
- classpath/embulk-output-postgresql-0.6.
|
35
|
-
- classpath/embulk-output-redshift-0.6.
|
34
|
+
- classpath/embulk-output-jdbc-0.6.2.jar
|
35
|
+
- classpath/embulk-output-postgresql-0.6.2.jar
|
36
|
+
- classpath/embulk-output-redshift-0.6.2.jar
|
36
37
|
- classpath/httpclient-4.3.6.jar
|
37
38
|
- classpath/httpcore-4.3.3.jar
|
38
39
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
Binary file
|
Binary file
|
Binary file
|