embulk-output-s3v2 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dcc56a0b48b656da07bac95bf806ddfb834ec62decd8f0cbf7afeb461b8d89bd
-  data.tar.gz: 4689c8b5fafdc1444fa3851dea5e809885d3ee019c8f4ba2c99e97128339e931
+  metadata.gz: 38c87b938e84dcad47a5b47f65872b15d6a549877003bc1379bf9ca32c44eddb
+  data.tar.gz: 3505a9c76d0c385e25d8b1b11676143cca4b3c798d130a0e0c0724abd63dc1a1
 SHA512:
-  metadata.gz: 4d2b74adcc5bffc412fd1b6066d55df546f781d9e04b02ec5bfd69ec0cdc1196aeb120ee778ded69512fb588ca5f6b0c7e51a8b0b50ba0853979b36f77a1e47c
-  data.tar.gz: ea66f45405bd13af9de2a375d53a02d18ca0d6e93a08f25d2b542661381bbe9063bbdf2b5932efe8f212eb7707ebd7187dcabe880ca24a9d720113b8a7340416
+  metadata.gz: bc999df40b640047ba878092a623b22ad25b8c975600a4615165446703b3827e823d24ed7df17b7bcc1aa64ef181847414ef72381a529b41642573a868e0c045
+  data.tar.gz: 3d3b4f6360528d541fff6d84963dd96b4d44360c0d9ab0f47f657ec0fee550858ff03faa64f4c0d0f2b30ee8910516a497a43b6a2ba38ae4bf67cd8272caa848

data/README.md

@@ -13,6 +13,10 @@ Files stores on Amazon S3.
 
 ## Configuration
 - **region**: AWS region name. (string, required)
+- **enable_profile**: If true, AWS credentials profile will be used when authenticating AWS. (boolean, default: `false`)
+  - Supported in v0.2.0 or later
+- **profile**: AWS credentials profile name. If `enable_profile` is false, this parameter will be ignored. (string, default: `default`)
+  - Supported in v0.2.0 or later
 - **bucket**: S3 bucket name. (string, required)
 - **object_key_prefix**: Prefix of S3 Objects key name. (string, required)
 - **enable_multi_part_upload**: If true, multipart upload will be enable. (boolean, default: `false`)

data/build.gradle

@@ -6,7 +6,7 @@ plugins {
 import com.github.jrubygradle.JRubyExec
 
 group 'com.github.ttksm'
-version '0.1.0'
+version '0.2.0'
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/s3v2/PluginTask.java

@@ -10,6 +10,14 @@ public interface PluginTask
     @Config("region")
     public String getRegion();
 
+    @Config("enable_profile")
+    @ConfigDefault("false")
+    public boolean getEnableProfile();
+
+    @Config("profile")
+    @ConfigDefault("\"default\"")
+    public String getProfile();
+
     @Config("bucket")
     public String getBucket();
 

data/src/main/java/org/embulk/output/s3v2/s3/S3ClientManager.java

@@ -16,9 +16,13 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 import org.embulk.output.s3v2.util.ChunksizeComputation;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
 import software.amazon.awssdk.core.sync.RequestBody;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
 import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
 import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
@@ -37,16 +41,26 @@ public class S3ClientManager
 {
     private final S3Client s3;
 
-    public S3ClientManager(String regionName)
+    public S3ClientManager(String regionName, boolean enableProfile, String profileName)
     {
+        S3ClientBuilder builder = S3Client.builder();
+
         Region region = Region.of(regionName);
         if (Region.regions().stream().noneMatch(o -> o.equals(region))) {
             throw new IllegalArgumentException("Not found aws region: " + regionName);
         }
+        builder = builder.region(region);
 
-        s3 = S3Client.builder()
-                .region(region)
-                .build();
+        AwsCredentialsProvider provider;
+        if (enableProfile) {
+            provider = ProfileCredentialsProvider.create(profileName);
+        }
+        else {
+            provider = DefaultCredentialsProvider.builder().profileName("").build();
+        }
+        builder = builder.credentialsProvider(provider);
+
+        s3 = builder.build();
     }
 
     /**

data/src/main/java/org/embulk/output/s3v2/strategy/AbstractStrategy.java

@@ -13,7 +13,7 @@ abstract class AbstractStrategy implements TransactionalFileOutput
 
     public AbstractStrategy(PluginTask task, int taskIndex)
     {
-        s3 = new S3ClientManager(task.getRegion());
+        s3 = new S3ClientManager(task.getRegion(), task.getEnableProfile(), task.getProfile());
         this.task = task;
         this.taskIndex = taskIndex;
 

data/src/test/java/org/embulk/output/s3v2/util/ThresholdComputationTests.java

@@ -0,0 +1,43 @@
+package org.embulk.output.s3v2.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.nio.ByteBuffer;
+
+/**
+ * @see ThresholdComputation
+ */
+@ExtendWith(MockitoExtension.class)
+public class ThresholdComputationTests
+{
+    @Test
+    @DisplayName("Threshold check returns true")
+    public void testThresholdCheckReturnsTrue() throws Exception
+    {
+        ByteBuffer bf = ByteBuffer.allocate(1000001);
+        Assertions.assertTrue(ThresholdComputation.largerThanThreshold("1MB", bf.capacity()));
+    }
+
+    @Test
+    @DisplayName("Threshold check returns false")
+    public void testThresholdCheckReturnsFalse() throws Exception
+    {
+        ByteBuffer bf = ByteBuffer.allocate(1000000);
+        Assertions.assertFalse(ThresholdComputation.largerThanThreshold("1MB", bf.capacity()));
+    }
+
+    @Test
+    @DisplayName("Invalid threshold string")
+    public void testInvalidThresholdString() throws Exception
+    {
+        ByteBuffer bf = ByteBuffer.allocate(1000000);
+        String threshold = "1MB1MB";
+        IllegalArgumentException ex = Assertions.assertThrows(IllegalArgumentException.class,
+                () -> ThresholdComputation.largerThanThreshold(threshold, bf.capacity()));
+        Assertions.assertEquals("Unrecognized value of multipart_threshold: " + threshold, ex.getMessage());
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3v2
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Toshihiro Takushima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-07 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies: []
 description: Stores files on Amazon S3 using aws-sdk-java-v2.
 email:
@@ -23,45 +23,45 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/annotations-2.15.60.jar
-- classpath/apache-client-2.15.60.jar
-- classpath/arns-2.15.60.jar
-- classpath/auth-2.15.60.jar
-- classpath/aws-core-2.15.60.jar
-- classpath/aws-query-protocol-2.15.60.jar
-- classpath/aws-xml-protocol-2.15.60.jar
+- classpath/annotations-2.15.61.jar
+- classpath/apache-client-2.15.61.jar
+- classpath/arns-2.15.61.jar
+- classpath/auth-2.15.61.jar
+- classpath/aws-core-2.15.61.jar
+- classpath/aws-query-protocol-2.15.61.jar
+- classpath/aws-xml-protocol-2.15.61.jar
 - classpath/commons-codec-1.11.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-output-s3v2-0.1.0.jar
+- classpath/embulk-output-s3v2-0.2.0.jar
 - classpath/eventstream-1.0.1.jar
-- classpath/http-client-spi-2.15.60.jar
+- classpath/http-client-spi-2.15.61.jar
 - classpath/httpclient-4.5.13.jar
 - classpath/httpcore-4.4.13.jar
 - classpath/jackson-annotations-2.10.5.jar
 - classpath/jackson-core-2.10.5.jar
 - classpath/jackson-databind-2.10.5.1.jar
-- classpath/metrics-spi-2.15.60.jar
+- classpath/metrics-spi-2.15.61.jar
 - classpath/netty-buffer-4.1.53.Final.jar
 - classpath/netty-codec-4.1.53.Final.jar
 - classpath/netty-codec-http-4.1.53.Final.jar
 - classpath/netty-codec-http2-4.1.53.Final.jar
 - classpath/netty-common-4.1.53.Final.jar
 - classpath/netty-handler-4.1.53.Final.jar
-- classpath/netty-nio-client-2.15.60.jar
+- classpath/netty-nio-client-2.15.61.jar
 - classpath/netty-reactive-streams-2.0.4.jar
 - classpath/netty-reactive-streams-http-2.0.4.jar
 - classpath/netty-resolver-4.1.53.Final.jar
 - classpath/netty-transport-4.1.53.Final.jar
 - classpath/netty-transport-native-epoll-4.1.53.Final-linux-x86_64.jar
 - classpath/netty-transport-native-unix-common-4.1.53.Final.jar
-- classpath/profiles-2.15.60.jar
-- classpath/protocol-core-2.15.60.jar
+- classpath/profiles-2.15.61.jar
+- classpath/protocol-core-2.15.61.jar
 - classpath/reactive-streams-1.0.3.jar
-- classpath/regions-2.15.60.jar
-- classpath/s3-2.15.60.jar
-- classpath/sdk-core-2.15.60.jar
+- classpath/regions-2.15.61.jar
+- classpath/s3-2.15.61.jar
+- classpath/sdk-core-2.15.61.jar
 - classpath/slf4j-api-1.7.28.jar
-- classpath/utils-2.15.60.jar
+- classpath/utils-2.15.61.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar
@@ -82,6 +82,7 @@ files:
 - src/main/java/org/embulk/output/s3v2/util/ThresholdComputation.java
 - src/test/java/org/embulk/output/s3v2/strategy/BufferedStrategyTests.java
 - src/test/java/org/embulk/output/s3v2/util/ChunksizeComputationTests.java
+- src/test/java/org/embulk/output/s3v2/util/ThresholdComputationTests.java
 - src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker
 homepage: https://github.com/ttksm/embulk-output-s3v2
 licenses: