embulk-input-s3 0.2.10 → 0.2.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 055d297441e872f1cd700b295bbb920e3af77231
-  data.tar.gz: ffc29906648b0499922cf84c62989414689811e3
+  metadata.gz: 034a5456271657abde4e9dc5aac76f7e05efc7e4
+  data.tar.gz: 74f38ae4046a4292487361a81d74c3be201b5aaa
 SHA512:
-  metadata.gz: cf0e4b463a3319320bc0dfdc5f5cf7297f1de646134b2e6c985fed3e41eee1f382bcd91d2e44d7063fedc777b1c9172abec71d5e3faa096ff586d64e86eea860
-  data.tar.gz: d0c99424fdb1c174637cf07330f72286e43c4f793ee77471d6d0bfb86f3186ca60917d732bec85c16da511e1297363488ea6c4a4d5b7384fc5b6782f649390d1
+  metadata.gz: 3cc77fc7cf409ce21e2a7373fc983727953a0d22b3c5676c43f4a4c3182a15de487ec3633d3b7b037386451317b5ea5ee2861ce46ae22aff66e1ef0a462b4f42
+  data.tar.gz: 19ff469baf507e141f91a17161d7bf2b48172e273b87b1ef1abb2464c385dc7ea7ced75e64baf04928295bce218203d5f48d2ecedd718efd7291ba53693c78c0

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -43,6 +43,9 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
+
+import static com.amazonaws.Protocol.HTTP;
+import static com.amazonaws.Protocol.HTTPS;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public abstract class AbstractS3FileInputPlugin
@@ -67,6 +70,11 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("null")
         public Optional<String> getAccessKeyId();
 
+        @Config("http_proxy")
+        @ConfigDefault("null")
+        public Optional<HttpProxy> getHttpProxy();
+        public void setHttpProxy(Optional<HttpProxy> httpProxy);
+
         @Config("incremental")
         @ConfigDefault("true")
         public boolean getIncremental();
@@ -144,9 +152,38 @@ public abstract class AbstractS3FileInputPlugin
         clientConfig.setMaxErrorRetry(3); // SDK default: 3
         clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
 
+        // set http proxy
+        if (task.getHttpProxy().isPresent()) {
+            setHttpProxyInAwsClient(clientConfig, task.getHttpProxy().get());
+        }
+
         return clientConfig;
     }
 
+    private void setHttpProxyInAwsClient(ClientConfiguration clientConfig, HttpProxy httpProxy)
+    {
+        // host
+        clientConfig.setProxyHost(httpProxy.getHost());
+
+        // port
+        if (httpProxy.getPort().isPresent()) {
+            clientConfig.setProxyPort(httpProxy.getPort().get());
+        }
+
+        // https
+        clientConfig.setProtocol(httpProxy.getHttps() ? Protocol.HTTPS : Protocol.HTTP);
+
+        // user
+        if (httpProxy.getUser().isPresent()) {
+            clientConfig.setProxyUsername(httpProxy.getUser().get());
+        }
+
+        // password
+        if (httpProxy.getPassword().isPresent()) {
+            clientConfig.setProxyPassword(httpProxy.getPassword().get());
+        }
+    }
+
     private FileList listFiles(PluginTask task)
     {
         try {

data/src/main/java/org/embulk/input/s3/HttpProxy.java

@@ -0,0 +1,35 @@
+package org.embulk.input.s3;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+
+/**
+ * HttpProxy is config unit for Input/Output plugins' configs.
+ *
+ * TODO: This unit will be moved to embulk/embulk-plugin-units.git.
+ * TODO: Consider using @JsonProperty(defaultValue=...) in Jackson 2.6+.
+ */
+public interface HttpProxy
+    extends Task
+{
+    @Config("host")
+    public String getHost();
+
+    @Config("port")
+    @ConfigDefault("null")
+    public Optional<Integer> getPort();
+
+    @Config("https")
+    @ConfigDefault("true")
+    public boolean getHttps();
+
+    @Config("user")
+    @ConfigDefault("null")
+    public Optional<String> getUser();
+
+    @Config("password")
+    @ConfigDefault("null")
+    public Optional<String> getPassword();
+}

data/src/test/java/org/embulk/input/s3/TestHttpProxy.java

@@ -0,0 +1,120 @@
+package org.embulk.input.s3;
+
+import com.google.common.base.Optional;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestHttpProxy
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+
+    @Before
+    public void createResources()
+    {
+        config = runtime.getExec().newConfigSource();
+        setupS3Config(config);
+    }
+
+    @Test
+    public void checkDefaultHttpProxy()
+    {
+        ConfigSource conf = config.deepCopy();
+        setupS3Config(conf);
+        S3PluginTask task = conf.loadConfig(S3PluginTask.class);
+        assertTrue(!task.getHttpProxy().isPresent());
+    }
+
+    @Test
+    public void checkHttpProxy()
+    {
+        { // specify host
+            String host = "my_host";
+            ConfigSource conf = config.deepCopy().set("host", host);
+            HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
+            assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
+                    httpProxy);
+        }
+
+        { // specify https=true explicitly
+            String host = "my_host";
+            ConfigSource conf = config.deepCopy()
+                    .set("host", host)
+                    .set("https", true);
+            HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
+            assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
+                    httpProxy);
+        }
+
+        { // specify https=false
+            String host = "my_host";
+            ConfigSource conf = config.deepCopy()
+                    .set("host", host)
+                    .set("https", false);
+            HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
+            assertHttpProxy(host, Optional.<Integer>absent(), false, Optional.<String>absent(), Optional.<String>absent(),
+                    httpProxy);
+        }
+
+        { // specify host, port
+            String host = "my_host";
+            int port = 8080;
+            ConfigSource conf = config.deepCopy()
+                    .set("host", host)
+                    .set("port", port);
+            HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
+            assertHttpProxy(host, Optional.of(port), true, Optional.<String>absent(), Optional.<String>absent(),
+                    httpProxy);
+        }
+
+        { // specify host, port, user, password
+            String host = "my_host";
+            int port = 8080;
+            String user = "my_user";
+            String password = "my_pass";
+            ConfigSource conf = config.deepCopy()
+                    .set("host", host)
+                    .set("port", port)
+                    .set("user", user)
+                    .set("password", password);
+            HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
+            assertHttpProxy(host, Optional.of(port), true, Optional.of(user), Optional.of(password),
+                    httpProxy);
+        }
+    }
+
+    private static void setupS3Config(ConfigSource config)
+    {
+        config.set("bucket", "my_bucket").set("path_prefix", "my_path_prefix");
+    }
+
+    private static void assertHttpProxy(String host, Optional<Integer> port, boolean https, Optional<String> user, Optional<String> password,
+            HttpProxy actual)
+    {
+        assertEquals(host, actual.getHost());
+        assertEquals(port.isPresent(), actual.getPort().isPresent());
+        if (port.isPresent()) {
+            assertEquals(port.get(), actual.getPort().get());
+        }
+
+        assertEquals(https, actual.getHttps());
+
+        assertEquals(user.isPresent(), actual.getUser().isPresent());
+        if (user.isPresent()) {
+            assertEquals(user.get(), actual.getUser().get());
+        }
+        assertEquals(password.isPresent(), actual.getPassword().isPresent());
+        if (password.isPresent()) {
+            assertEquals(password.get(), actual.getPassword().get());
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.10
+  version: 0.2.11
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-09 00:00:00.000000000 Z
+date: 2017-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -49,9 +49,11 @@ files:
 - lib/embulk/input/s3.rb
 - src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
 - src/main/java/org/embulk/input/s3/FileList.java
+- src/main/java/org/embulk/input/s3/HttpProxy.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
+- src/test/java/org/embulk/input/s3/TestHttpProxy.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
 - src/test/resources/sample_01.csv
@@ -59,8 +61,8 @@ files:
 - classpath/aws-java-sdk-kms-1.10.33.jar
 - classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.10.jar
-- classpath/embulk-util-aws-credentials-0.2.10.jar
+- classpath/embulk-input-s3-0.2.11.jar
+- classpath/embulk-util-aws-credentials-0.2.11.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/jcl-over-slf4j-1.7.12.jar