embulk-output-s3_per_record 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88cdda711509c0f8890ae1dfa9c9720c1067bd0b
-  data.tar.gz: 1618f1e1d874773f3edcaa738bd1f4bd4d4c766e
+  metadata.gz: 66c630852cd7438c47f5c7d4ce90e5ae9c260c28
+  data.tar.gz: 7485c4a4d693740fb7f6ef9975d31ab3f63522fc
 SHA512:
-  metadata.gz: 1b2b9029f619efcb751886b52d755cfd8bf11750a87c6a98c4f9561a539083ec78ece754fe1d2fa19be646fb2d3d677488207b2b1cbb31c4405b181c56bd5c53
-  data.tar.gz: 1c39b5d32160d7e2290bba51c376971131f44a5d81159db285340119c1d28970d984d05d5ba2dc8c569189189e242722241136965a8e92c6926cd5143d75d9f6
+  metadata.gz: d05834b8904544d92f524e09919dd8fc6793da6243a76c43056da0d8c16d025534ab5d4d036c63625acd61545a907e8bda4431312f072f66df66cdde8dc42858
+  data.tar.gz: 9181a74cc5ccd62a19090d864f96697ee74b7c0b4defeb2158b51fe4f02f7c9c89b146b105d67ecf34efc2b567d290fb7aaa84b29b69aa0726e5a714f26bc406

data/README.md

@@ -16,6 +16,7 @@ S3 object key can be composed of another column.
 - **data_column**: Column for object's body.
 - **aws_access_key_id**: (optional) AWS access key id. If not given, [DefaultAWSCredentialsProviderChain](http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) is used to get credentials.
 - **aws_secret_access_key**: (optional) AWS secret access key. Required if `aws_access_key_id` is given.
+- **base64**: (default false) If true, decode the value as Base64 before uploading.
 
 ## Example
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.output.s3_per_record;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -29,6 +30,7 @@ import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.Upload;
+import com.amazonaws.util.Base64;
 import com.google.common.base.Optional;
 
 
@@ -63,6 +65,11 @@ public class S3PerRecordOutputPlugin
         @Config("aws_secret_access_key")
         @ConfigDefault("null")
         Optional<String> getAwsSecretAccessKey();
+
+        // Enable Base64 decoding
+        @Config("base64")
+        @ConfigDefault("false")
+        boolean getBase64();
     }
 
     @Override
@@ -107,12 +114,14 @@ public class S3PerRecordOutputPlugin
         private final Column dataColumn;
         private final Schema schema;
         private List<Upload> uploads;
+        private final boolean decodeBase64;
 
         public S3PerRecordPageOutput(PluginTask task, Schema schema) {
             this.schema = schema;
             bucket = task.getBucket();
             keyPattern = makeKeyPattern(task.getKey());
             dataColumn = schema.lookupColumn(task.getDataColumn());
+            decodeBase64 = task.getBase64();
 
             AWSCredentials credentials;
             if (task.getAwsAccessKeyId().isPresent() && task.getAwsSecretAccessKey().isPresent()) {
@@ -157,7 +166,12 @@ public class S3PerRecordOutputPlugin
                 String key = buildKey(pageReader);
 
                 String payload = pageReader.getString(dataColumn);
-                byte[] payloadBytes = payload.getBytes();
+                byte[] payloadBytes;
+                if (decodeBase64) {
+                    payloadBytes = Base64.decode(payload);
+                } else {
+                    payloadBytes = payload.getBytes(StandardCharsets.UTF_8);
+                }
                 ObjectMetadata metadata = new ObjectMetadata();
                 metadata.setContentLength(payloadBytes.length);
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3_per_record
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - tomykaira
@@ -64,7 +64,7 @@ files:
 - classpath/commons-codec-1.6.jar
 - classpath/aws-java-sdk-s3-1.10.71.jar
 - classpath/aws-java-sdk-core-1.10.71.jar
-- classpath/embulk-output-s3_per_record-0.1.0.jar
+- classpath/embulk-output-s3_per_record-0.1.1.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/aws-java-sdk-kms-1.10.71.jar
 homepage: https://github.com/tomykaira/embulk-output-s3_per_record