embulk-input-http 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0fd9d1c50eb1690a68282058341d8dfe47c2706
-  data.tar.gz: e64cca9c2d869cbd0a107331aad8ca91e4187441
+  metadata.gz: 0be0755b221d7db3033f0c0193aecddb989ab46f
+  data.tar.gz: 1d027f7fca3f58087e93bc922a271f0126864842
 SHA512:
-  metadata.gz: 72182010d6bb889cc86203e466d1cdcf35a64d47e4e38167414b7d204320558e7659aa3c09c0448e679c10a41a6e9884b278e63b545a4271d3b6f10571222760
-  data.tar.gz: cfe7a025c51abc126e4837578815b2139f7c938a0ee41e32f3a5d919589fdb1f03d18c61fa768b299bcd6d2258154dc764ff768c700de6c59c966d14e77bd782
+  metadata.gz: 1f520102f0a5df0f642e9789e3fe035b6007e2bef3bfc3a3d57d816c7989e80dac7e6340455b4cfd802150033ac5b9c7737e3914da5aff7cfedd4dd96b631938
+  data.tar.gz: ace4460f528b39465978d4753492ab9ad259c33cd3c7373df1d5583907592d3a067cf6052ed9b5d87156a0fc4d10b92e4eaf7240a4d3ceb27bdda451f736110b

data/build.gradle

@@ -12,12 +12,12 @@ configurations {
     provided
 }
 
-version = "0.0.6"
+version = "0.0.7"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.5.2"
+    compile  "org.embulk:embulk-core:0.6.8"
     compile  "org.apache.httpcomponents:httpclient:4.4"
-    provided "org.embulk:embulk-core:0.5.2"
+    provided "org.embulk:embulk-core:0.6.8"
     testCompile "junit:junit:4.+"
 }
 

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -4,14 +4,11 @@ import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
 import org.apache.http.Header;
-import org.apache.http.HttpException;
-import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthScope;
 import org.apache.http.auth.UsernamePasswordCredentials;
 import org.apache.http.client.CredentialsProvider;
 import org.apache.http.client.HttpClient;
-import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
@@ -22,13 +19,13 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.message.BasicNameValuePair;
-import org.apache.http.util.EntityUtils;
 import org.embulk.config.*;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -162,16 +159,10 @@ public class HttpInputPlugin implements FileInputPlugin {
         }
 
         HttpClientBuilder builder = HttpClientBuilder.create()
+                .disableAutomaticRetries()
                 .setDefaultRequestConfig(makeRequestConfig(task))
                 .setDefaultHeaders(makeHeaders(task));
 
-        if (task.getMaxRetries() > 0) {
-            final int retry = task.getMaxRetries();
-            final int interval = task.getRetryInterval();
-            HttpRequestRetryHandler retryHandler = new RetryHandler(retry, interval);
-            builder.setRetryHandler(retryHandler);
-        }
-
         if (task.getBasicAuth().isPresent()) {
             builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(),
                     request));
@@ -179,28 +170,28 @@ public class HttpInputPlugin implements FileInputPlugin {
 
         HttpClient client = builder.build();
 
-        if (task.getSleepBeforeRequest() > 0) {
-            try {
-                logger.info(String.format("Waiting %d msec ...", task.getSleepBeforeRequest()));
-                Thread.sleep(task.getSleepBeforeRequest());
-            } catch (InterruptedException e) {
-            }
-        }
-
         logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
                 request.getURI().toString()));
+
+        RetryableHandler retryable = new RetryableHandler(client, request);
         try {
-            HttpResponse response = client.execute(request);
-            statusIsOkOrThrow(response);
-            //final String body = EntityUtils.toString(response.getEntity());
-            //InputStream stream = new ByteArrayInputStream(body.getBytes());
-            InputStream stream = response.getEntity().getContent();
+            if (task.getSleepBeforeRequest() > 0) {
+                logger.info(String.format("wait %d msec before request", task.getSleepBeforeRequest()));
+                Thread.sleep(task.getSleepBeforeRequest());
+            }
+            RetryExecutor.retryExecutor().
+                    withRetryLimit(task.getMaxRetries()).
+                    withInitialRetryWait(task.getRetryInterval()).
+                    withMaxRetryWait(30 * 60 * 1000).
+                    runInterruptible(retryable);
+            InputStream stream = retryable.getResponse().getEntity().getContent();
             PluginFileInput input = new PluginFileInput(task, stream);
             stream = null;
             return input;
-        } catch (IOException | HttpException e) {
+        } catch (Exception e) {
             throw Throwables.propagate(e);
         }
+
     }
 
     private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
@@ -264,18 +255,6 @@ public class HttpInputPlugin implements FileInputPlugin {
                 .build();
     }
 
-    private void statusIsOkOrThrow(HttpResponse response)
-            throws HttpException, IOException {
-        int code = response.getStatusLine().getStatusCode();
-        switch (response.getStatusLine().getStatusCode()) {
-            case 200:
-                return;
-            default:
-                throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
-                        code, EntityUtils.toString(response.getEntity())));
-        }
-    }
-
     public static class PluginFileInput extends InputStreamFileInput
             implements TransactionalFileInput {
 

data/src/main/java/org/embulk/input/RetryableHandler.java

@@ -0,0 +1,90 @@
+package org.embulk.input;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.util.EntityUtils;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.UnknownHostException;
+import java.util.List;
+
+public class RetryableHandler implements RetryExecutor.Retryable {
+
+    protected final Logger logger = Exec.getLogger(getClass());
+
+    private static List<Class<? extends IOException>> NOT_RETRIABLE_CLAASSES;
+
+    private final HttpClient client;
+    private final HttpRequestBase request;
+    private HttpResponse response;
+
+    static {
+        ImmutableList.Builder<Class<? extends IOException>> classes = ImmutableList.builder();
+        classes.add(UnknownHostException.class).
+                add(InterruptedIOException.class).
+                add(SSLException.class);
+        NOT_RETRIABLE_CLAASSES = classes.build();
+    }
+
+    public RetryableHandler(HttpClient client, HttpRequestBase request) {
+        this.client = client;
+        this.request = request;
+    }
+
+    public HttpResponse getResponse() {
+        return response;
+    }
+
+    @Override
+    public Object call() throws Exception {
+        if (response != null) throw new IllegalStateException("response is already set");
+        HttpResponse response = client.execute(request);
+        statusIsOkOrThrow(response);
+        this.response = response;
+        return null;
+    }
+
+    @Override
+    public boolean isRetryableException(Exception exception) {
+        if (NOT_RETRIABLE_CLAASSES.contains(exception.getClass())) {
+            logger.error(String.format("'%s' is not retriable", exception.getClass()));
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            throws RetryExecutor.RetryGiveupException {
+        logger.warn("retrying {}/{} after {} seconds. Message: {}",
+                retryCount, retryLimit, retryWait / 1000,
+                exception.getMessage());
+    }
+
+    @Override
+    public void onGiveup(Exception firstException, Exception lastException)
+            throws RetryExecutor.RetryGiveupException {
+        logger.error("giveup {}", lastException.getMessage());
+    }
+
+    protected void statusIsOkOrThrow(HttpResponse response)
+            throws HttpException, IOException {
+        int code = response.getStatusLine().getStatusCode();
+        switch (response.getStatusLine().getStatusCode()) {
+            case 200:
+                return;
+            default:
+                throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
+                        code, EntityUtils.toString(response.getEntity())));
+        }
+    }
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-http
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Takuma kanari
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-13 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,15 +59,15 @@ files:
 - src/main/java/org/embulk/input/HttpInputPlugin.java
 - src/main/java/org/embulk/input/ParamsConfig.java
 - src/main/java/org/embulk/input/QueryConfig.java
-- src/main/java/org/embulk/input/RetryHandler.java
+- src/main/java/org/embulk/input/RetryableHandler.java
 - src/test/java/org/embulk/input/TestHttpInputPlugin.java
 - src/test/java/org/embulk/input/TestParamsConfig.java
 - src/test/java/org/embulk/input/TestQueryConfig.java
-- classpath/httpclient-4.4.jar
+- classpath/commons-codec-1.9.jar
 - classpath/commons-logging-1.2.jar
+- classpath/embulk-input-http-0.0.7.jar
+- classpath/httpclient-4.4.jar
 - classpath/httpcore-4.4.jar
-- classpath/commons-codec-1.9.jar
-- classpath/embulk-input-http-0.0.6.jar
 homepage: https://github.com/takumakanari/embulk-input-http
 licenses:
 - MIT

data/src/main/java/org/embulk/input/RetryHandler.java

@@ -1,44 +0,0 @@
-package org.embulk.input;
-
-import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
-import org.apache.http.protocol.HttpContext;
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
-
-import javax.net.ssl.SSLException;
-import java.io.IOException;
-import java.net.ConnectException;
-import java.net.UnknownHostException;
-import java.util.Arrays;
-
-public class RetryHandler extends DefaultHttpRequestRetryHandler
-{
-
-    private final Logger logger = Exec.getLogger(getClass());
-
-    private int interval = 0;
-
-    public RetryHandler(int retry, int interval)
-    {
-        super(retry, true, Arrays.asList(
-                UnknownHostException.class,
-                ConnectException.class,
-                SSLException.class));
-        this.interval = interval;
-    }
-
-    @Override
-    public boolean retryRequest(final IOException exception,
-                                final int executionCount, final HttpContext context)
-    {
-        final boolean isRetriable = super.retryRequest(exception, executionCount, context);
-        if (isRetriable) {
-            try {
-                logger.info(String.format("Sleep %d msec before retry", interval));
-                Thread.sleep(interval);
-            } catch (InterruptedException e) {}
-        }
-        return isRetriable;
-    }
-
-}