embulk-output-redshift 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d9ca4eb2942aae5f463bf611914d8cf76393ab7d
-  data.tar.gz: f65aa7a7ee215d13437567609be6233ed538f2dd
+  metadata.gz: 6fbe390d4e95de66a23275061c7841365a9d4f28
+  data.tar.gz: 2685fb58fccbbc8d26b1824a6d57cb9b1cba5d45
 SHA512:
-  metadata.gz: c0f0fc9b9270cff6e50456594bd41ea87b4e5f89f55faf0709dbc40263ebdf63497528174e759b901dfab7ad6e5bf024a84abf91472274cddbe691cbfe5bdba5
-  data.tar.gz: 136189a5e9b844cc5c9a8bb87a39e176a69db3544ee717735bbfd7e1190bba03bc4aa5c1df5cd7841457f7a07728b3f101af807cc196b93374b6bcaed27bfdbb
+  metadata.gz: 37171adae4aee6a663efdce2663ae087faa6549acb3f4f33ba2cad1cd1d871fd6e6934a84e1878699d82f7a9b84ccde74b8210548b929a3f4f08e19664908879
+  data.tar.gz: 924cba2f34150a1e87112f9e8277e796f70fb595418adfc8f8c654f71a1a58c9ae048c4ab2b616eb549bc86836f64ed9b7605a7e0b49c1997091d694c193d39c

data/README.md

@@ -1,97 +1,104 @@
-# Redshift output plugins for Embulk
-
-Redshift output plugins for Embulk loads records to Redshift.
-
-## Overview
-
-* **Plugin type**: output
-* **Load all or nothing**: depnds on the mode. see bellow.
-* **Resume supported**: depnds on the mode. see bellow.
-
-## Configuration
-
-- **host**: database host name (string, required)
-- **port**: database port number (integer, default: 5439)
-- **user**: database login user name (string, required)
-- **password**: database login password (string, default: "")
-- **database**: destination database name (string, required)
-- **schema**: destination schema name (string, default: "public")
-- **table**: destination table name (string, required)
-- **options**: extra connection properties (hash, default: {})
-- **mode**: "replace" or "insert" (string, required)
-- **batch_size**: size of a single batch insert (integer, default: 16777216)
-- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
-- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
-  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
-  - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
-  - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
-  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
-
-### Modes
-
-* **insert**:
-  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ...` query.
-  * Transactional: Yes. This mode successfully writes all rows, or fails with writing zero rows.
-  * Resumable: Yes.
-* **insert_direct**:
-  * Behavior: This mode inserts rows to the target table directly.
-  * Transactional: No. If fails, the target table could have some rows inserted.
-  * Resumable: No.
-* **truncate_insert**:
-  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
-  * Transactional: Yes.
-  * Resumable: Yes.
-* **merge**:
-  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query.
-  * Transactional: Yes.
-  * Resumable: Yes.
-* **replace**:
-  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
-  * Transactional: Yes.
-  * Resumable: No.
-
-### Example
-
-```yaml
-out:
-  type: redshift
-  host: myinstance.us-west-2.redshift.amazonaws.com
-  user: pg
-  password: ""
-  database: my_database
-  table: my_table
-  access_key_id: ABCXYZ123ABCXYZ123
-  secret_access_key: AbCxYz123aBcXyZ123
-  s3_bucket: my-redshift-transfer-bucket
-  iam_user_name: my-s3-read-only
-  mode: insert
-```
-
-Advanced configuration:
-
-```yaml
-out:
-  type: redshift
-  host: myinstance.us-west-2.redshift.amazonaws.com
-  user: pg
-  password: ""
-  database: my_database
-  table: my_table
-  access_key_id: ABCXYZ123ABCXYZ123
-  secret_access_key: AbCxYz123aBcXyZ123
-  s3_bucket: my-redshift-transfer-bucket
-  iam_user_name: my-s3-read-only
-  options: {loglevel: 2}
-  mode: insert_direct
-  column_options:
-    my_col_1: {type: 'VARCHAR(255)'}
-    my_col_3: {type: 'INT NOT NULL'}
-    my_col_4: {value_type: string, timestamp_format: `%Y-%m-%d %H:%M:%S %z`, timezone: '-0700'}
-    my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
-```
-
-### Build
-
-```
-$ ./gradlew gem
-```
+# Redshift output plugins for Embulk
+
+Redshift output plugins for Embulk loads records to Redshift.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depnds on the mode. see bellow.
+* **Resume supported**: depnds on the mode. see bellow.
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, default: 5439)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **schema**: destination schema name (string, default: "public")
+- **table**: destination table name (string, required)
+- **access_key_id**: access key id for AWS
+- **secret_access_key: secret access key for AWS
+- **iam_user_name: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:ListBucket` and `sts:GetFederationToken`.
+- **s3_bucket: S3 bucket name for temporary files
+- **s3_key_prefix: S3 key prefix for temporary files (string, default:"")
+- **options**: extra connection properties (hash, default: {})
+- **mode**: "replace" or "insert" (string, required)
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, truncate_insert and merge modes), when it creates the target table (insert_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
+  - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
+  - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+
+### Modes
+
+* **insert**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ...` query.
+  * Transactional: Yes. This mode successfully writes all rows, or fails with writing zero rows.
+  * Resumable: Yes.
+* **insert_direct**:
+  * Behavior: This mode inserts rows to the target table directly.
+  * Transactional: No. If fails, the target table could have some rows inserted.
+  * Resumable: No.
+* **truncate_insert**:
+  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
+  * Transactional: Yes.
+  * Resumable: Yes.
+* **merge**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query.
+  * Transactional: Yes.
+  * Resumable: Yes.
+* **replace**:
+  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
+  * Transactional: Yes.
+  * Resumable: No.
+
+### Example
+
+```yaml
+out:
+  type: redshift
+  host: myinstance.us-west-2.redshift.amazonaws.com
+  user: pg
+  password: ""
+  database: my_database
+  table: my_table
+  access_key_id: ABCXYZ123ABCXYZ123
+  secret_access_key: AbCxYz123aBcXyZ123
+  iam_user_name: my-s3-read-only
+  s3_bucket: my-redshift-transfer-bucket
+  s3_key_prefix: temp/redshift
+  mode: insert
+```
+
+Advanced configuration:
+
+```yaml
+out:
+  type: redshift
+  host: myinstance.us-west-2.redshift.amazonaws.com
+  user: pg
+  password: ""
+  database: my_database
+  table: my_table
+  access_key_id: ABCXYZ123ABCXYZ123
+  secret_access_key: AbCxYz123aBcXyZ123
+  iam_user_name: my-s3-read-only
+  s3_bucket: my-redshift-transfer-bucket
+  s3_key_prefix: temp/redshift
+  options: {loglevel: 2}
+  mode: insert_direct
+  column_options:
+    my_col_1: {type: 'VARCHAR(255)'}
+    my_col_3: {type: 'INT NOT NULL'}
+    my_col_4: {value_type: string, timestamp_format: `%Y-%m-%d %H:%M:%S %z`, timezone: '-0700'}
+    my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
+```
+
+### Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -1,9 +1,9 @@
-dependencies {
-    compile project(':embulk-output-jdbc')
-    compile project(':embulk-output-postgresql')
-
-    compile "com.amazonaws:aws-java-sdk-s3:1.9.17"
-    compile "com.amazonaws:aws-java-sdk-sts:1.9.17"
-
-    testCompile project(':embulk-output-jdbc').sourceSets.test.output
-}
+dependencies {
+    compile project(':embulk-output-jdbc')
+    compile project(':embulk-output-postgresql')
+
+    compile "com.amazonaws:aws-java-sdk-s3:1.9.17"
+    compile "com.amazonaws:aws-java-sdk-sts:1.9.17"
+
+    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+}

data/lib/embulk/output/redshift.rb

@@ -1,3 +1,3 @@
-Embulk::JavaPlugin.register_output(
-  :redshift, "org.embulk.output.RedshiftOutputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))
+Embulk::JavaPlugin.register_output(
+  :redshift, "org.embulk.output.RedshiftOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -1,147 +1,151 @@
-package org.embulk.output;
-
-import java.util.List;
-import java.util.Properties;
-import java.io.IOException;
-import java.sql.SQLException;
-import org.slf4j.Logger;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import org.embulk.spi.Exec;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
-import org.embulk.output.jdbc.BatchInsert;
-import org.embulk.output.redshift.RedshiftOutputConnector;
-import org.embulk.output.redshift.RedshiftCopyBatchInsert;
-
-public class RedshiftOutputPlugin
-        extends AbstractJdbcOutputPlugin
-{
-    private final Logger logger = Exec.getLogger(RedshiftOutputPlugin.class);
-
-    public interface RedshiftPluginTask extends PluginTask
-    {
-        @Config("host")
-        public String getHost();
-
-        @Config("port")
-        @ConfigDefault("5439")
-        public int getPort();
-
-        @Config("user")
-        public String getUser();
-
-        @Config("password")
-        @ConfigDefault("\"\"")
-        public String getPassword();
-
-        @Config("database")
-        public String getDatabase();
-
-        @Config("schema")
-        @ConfigDefault("\"public\"")
-        public String getSchema();
-
-        @Config("access_key_id")
-        public String getAccessKeyId();
-
-        @Config("secret_access_key")
-        public String getSecretAccessKey();
-
-        @Config("iam_user_name")
-        public String getIamUserName();
-
-        @Config("s3_bucket")
-        public String getS3Bucket();
-    }
-
-    @Override
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return RedshiftPluginTask.class;
-    }
-
-    @Override
-    protected Features getFeatures(PluginTask task)
-    {
-        return new Features()
-            .setMaxTableNameLength(30)
-            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
-            .setIgnoreMergeKeys(false);
-    }
-
-    @Override
-    protected RedshiftOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
-    {
-        RedshiftPluginTask t = (RedshiftPluginTask) task;
-
-        String url = String.format("jdbc:postgresql://%s:%d/%s",
-                t.getHost(), t.getPort(), t.getDatabase());
-
-        Properties props = new Properties();
-        props.setProperty("loginTimeout",   "300"); // seconds
-        props.setProperty("socketTimeout", "1800"); // seconds
-
-        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
-        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
-        props.setProperty("tcpKeepAlive", "true");
-
-        // TODO
-        //switch task.getSssl() {
-        //when "disable":
-        //    break;
-        //when "enable":
-        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
-        //when "verify":
-        //    props.setProperty("ssl", "true");
-        //    break;
-        //}
-
-        if (!retryableMetadataOperation) {
-            // non-retryable batch operation uses longer timeout
-            props.setProperty("loginTimeout",    "300");  // seconds
-            props.setProperty("socketTimeout", "28800");  // seconds
-        }
-
-        props.putAll(t.getOptions());
-
-        props.setProperty("user", t.getUser());
-        logger.info("Connecting to {} options {}", url, props);
-        props.setProperty("password", t.getPassword());
-
-        return new RedshiftOutputConnector(url, props, t.getSchema());
-    }
-
-    private static AWSCredentialsProvider getAWSCredentialsProvider(RedshiftPluginTask task)
-    {
-        final AWSCredentials creds = new BasicAWSCredentials(
-                task.getAccessKeyId(), task.getSecretAccessKey());
-        return new AWSCredentialsProvider() {
-            @Override
-            public AWSCredentials getCredentials()
-            {
-                return creds;
-            }
-
-            @Override
-            public void refresh()
-            {
-            }
-        };
-    }
-
-    @Override
-    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
-    {
-        if (mergeKeys.isPresent()) {
-            throw new UnsupportedOperationException("Redshift output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
-        }
-        RedshiftPluginTask t = (RedshiftPluginTask) task;
-        return new RedshiftCopyBatchInsert(getConnector(task, true),
-                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getIamUserName());
-    }
-}
+package org.embulk.output;
+
+import java.util.List;
+import java.util.Properties;
+import java.io.IOException;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import org.embulk.spi.Exec;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.redshift.RedshiftOutputConnector;
+import org.embulk.output.redshift.RedshiftCopyBatchInsert;
+
+public class RedshiftOutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    private final Logger logger = Exec.getLogger(RedshiftOutputPlugin.class);
+
+    public interface RedshiftPluginTask extends PluginTask
+    {
+        @Config("host")
+        public String getHost();
+
+        @Config("port")
+        @ConfigDefault("5439")
+        public int getPort();
+
+        @Config("user")
+        public String getUser();
+
+        @Config("password")
+        @ConfigDefault("\"\"")
+        public String getPassword();
+
+        @Config("database")
+        public String getDatabase();
+
+        @Config("schema")
+        @ConfigDefault("\"public\"")
+        public String getSchema();
+
+        @Config("access_key_id")
+        public String getAccessKeyId();
+
+        @Config("secret_access_key")
+        public String getSecretAccessKey();
+
+        @Config("iam_user_name")
+        public String getIamUserName();
+
+        @Config("s3_bucket")
+        public String getS3Bucket();
+
+        @Config("s3_key_prefix")
+        @ConfigDefault("\"\"")
+        public String getS3KeyPrefix();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return RedshiftPluginTask.class;
+    }
+
+    @Override
+    protected Features getFeatures(PluginTask task)
+    {
+        return new Features()
+            .setMaxTableNameLength(30)
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setIgnoreMergeKeys(false);
+    }
+
+    @Override
+    protected RedshiftOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        RedshiftPluginTask t = (RedshiftPluginTask) task;
+
+        String url = String.format("jdbc:postgresql://%s:%d/%s",
+                t.getHost(), t.getPort(), t.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("loginTimeout",   "300"); // seconds
+        props.setProperty("socketTimeout", "1800"); // seconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch task.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory");  // disable server-side validation
+        //when "verify":
+        //    props.setProperty("ssl", "true");
+        //    break;
+        //}
+
+        if (!retryableMetadataOperation) {
+            // non-retryable batch operation uses longer timeout
+            props.setProperty("loginTimeout",    "300");  // seconds
+            props.setProperty("socketTimeout", "28800");  // seconds
+        }
+
+        props.putAll(t.getOptions());
+
+        props.setProperty("user", t.getUser());
+        logger.info("Connecting to {} options {}", url, props);
+        props.setProperty("password", t.getPassword());
+
+        return new RedshiftOutputConnector(url, props, t.getSchema());
+    }
+
+    private static AWSCredentialsProvider getAWSCredentialsProvider(RedshiftPluginTask task)
+    {
+        final AWSCredentials creds = new BasicAWSCredentials(
+                task.getAccessKeyId(), task.getSecretAccessKey());
+        return new AWSCredentialsProvider() {
+            @Override
+            public AWSCredentials getCredentials()
+            {
+                return creds;
+            }
+
+            @Override
+            public void refresh()
+            {
+            }
+        };
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    {
+        if (mergeKeys.isPresent()) {
+            throw new UnsupportedOperationException("Redshift output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
+        }
+        RedshiftPluginTask t = (RedshiftPluginTask) task;
+        return new RedshiftCopyBatchInsert(getConnector(task, true),
+                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName());
+    }
+}