embulk-input-http 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a5521bcc6ec05740477ee1d0b1a4771057563ce
-  data.tar.gz: dd85ad0a597122ffb6b666ded12d25ba05c8dcb0
+  metadata.gz: c03f483d0ec3c8838d1538940f9a206c0e3b58f8
+  data.tar.gz: 6e1e2d691f5d5acd099f12edb1cf6a3986a190ab
 SHA512:
-  metadata.gz: dbb23315e384a59799caaa502af1e6c8f51a0884bc74b2b7ce43eb6f96fff81a9ff1b5958a865054f626eab8c6a623bcf3567d8f83f56334f7099dbb0ab8fe97
-  data.tar.gz: 5f2f1c12f916e9847a4d3dc5a4c1cef760af8fc925b54c11e87d5492870e3ff56a0d01f97d4d9d8a2349713add5e58c2ed633bfde6d5672105f5ab70a4e84340
+  metadata.gz: c32fe0519c6d8444291cd7eb552256ae828106c160616490510d780fd1583cb98c482cc7351950dcc733d8e8c7f38aa378619c26a4b51317adffd086abb371c8
+  data.tar.gz: 518387033e044784f908c95e6af973882df9ab68da75066c5eb10b9b07151094c370b44ad9cbb878e3dd92efd907e18ba4ac9b635749bce7be6cf8964234db63

data/README.md

@@ -4,10 +4,6 @@ Input HTTP plugin for [Embulk](https://github.com/embulk/embulk).
 Fetch data via HTTP.
 
 
-### Big changes in v0.0.4
-
-From `v0.0.4`, *iterate* section is removed, embulk-input-http is must be used with some **parer plugin**.
-
 ## Installation
 
 Run this command with your embulk binary.
@@ -40,7 +36,9 @@ in:
 - **charset**: Charset to specify request header (optional, utf-8 is used by default)
 - **open_timeout**: timeout msec to open connection (optional, 2000 is used by default)
 - **read_timeout**: timeout msec to read content via http (optional, 10000 is used by default)
-
+- **max_retries**: max number of retry request if failed (optional, 5 is used by default)
+- **retry_interval**: interval msec to retry max (optional, 10000 is used by default)
+- **sleep\_before\_request**: wait msec before each requests (optional, 0 is used by default)
 
 ### Brace expansion style in params
 

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.0.4"
+version = "0.0.5"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.5.2"

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -5,8 +5,12 @@ import com.google.common.base.Throwables;
 import org.apache.http.Header;
 import org.apache.http.HttpException;
 import org.apache.http.HttpResponse;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.util.EntityUtils;
 import org.apache.http.NameValuePair;
 import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
 import org.apache.http.client.methods.HttpGet;
@@ -27,6 +31,7 @@ import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.ByteArrayInputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.URISyntaxException;
 import java.util.ArrayList;
@@ -60,6 +65,19 @@ public class HttpInputPlugin implements FileInputPlugin {
         @ConfigDefault("10000")
         public int getReadTimeout();
 
+        @Config("max_retries")
+        @ConfigDefault("5")
+        public int getMaxRetries();
+
+        @Config("retry_interval")
+        @ConfigDefault("10000")
+        public int getRetryInterval();
+
+        @Config("sleep_before_request")
+        @ConfigDefault("0")
+        public int getSleepBeforeRequest();
+        public void setSleepBeforeRequest(int sleepBeforeRequest);
+
         @Config("params")
         @ConfigDefault("null")
         public Optional<ParamsConfig> getParams();
@@ -92,6 +110,10 @@ public class HttpInputPlugin implements FileInputPlugin {
             task.setQueries(new ArrayList<ParamsConfig>());
         }
 
+        if (numOfThreads == 1) {
+            task.setSleepBeforeRequest(0);
+        }
+
         switch (task.getMethod().toUpperCase()) {
             case "GET":
                 task.setHttpMethod(HttpMethod.GET);
@@ -130,16 +152,35 @@ public class HttpInputPlugin implements FileInputPlugin {
         } catch (URISyntaxException | UnsupportedEncodingException e) {
             throw Throwables.propagate(e);
         }
-        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
-                request.getURI().toString()));
 
-        HttpClient client = HttpClientBuilder.create()
+        HttpClientBuilder builder = HttpClientBuilder.create()
                 .setDefaultRequestConfig(makeRequestConfig(task))
-                .setDefaultHeaders(makeHeaders(task))
-                .build();
+                .setDefaultHeaders(makeHeaders(task));
+
+        if (task.getMaxRetries() > 0) {
+            final int retry = task.getMaxRetries();
+            final int interval = task.getRetryInterval();
+            HttpRequestRetryHandler retryHandler = new RetryHandler(retry, interval);
+            builder.setRetryHandler(retryHandler);
+        }
+
+        HttpClient client = builder.build();
+
+        if (task.getSleepBeforeRequest() > 0) {
+            try {
+                logger.info(String.format("Waiting %d msec ...", task.getSleepBeforeRequest()));
+                Thread.sleep(task.getSleepBeforeRequest());
+            } catch (InterruptedException e) {
+            }
+        }
+
+        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
+                request.getURI().toString()));
         try {
             HttpResponse response = client.execute(request);
             statusIsOkOrThrow(response);
+            //final String body = EntityUtils.toString(response.getEntity());
+            //InputStream stream = new ByteArrayInputStream(body.getBytes());
             InputStream stream = response.getEntity().getContent();
             PluginFileInput input = new PluginFileInput(task, stream);
             stream = null;

data/src/main/java/org/embulk/input/RetryHandler.java

@@ -0,0 +1,44 @@
+package org.embulk.input;
+
+import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
+import org.apache.http.protocol.HttpContext;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.UnknownHostException;
+import java.util.Arrays;
+
+public class RetryHandler extends DefaultHttpRequestRetryHandler
+{
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    private int interval = 0;
+
+    public RetryHandler(int retry, int interval)
+    {
+        super(retry, true, Arrays.asList(
+                UnknownHostException.class,
+                ConnectException.class,
+                SSLException.class));
+        this.interval = interval;
+    }
+
+    @Override
+    public boolean retryRequest(final IOException exception,
+                                final int executionCount, final HttpContext context)
+    {
+        final boolean isRertriable = super.retryRequest(exception, executionCount, context);
+        if (isRertriable) {
+            try {
+                logger.info(String.format("Sleep %d msec before retry", interval));
+                Thread.sleep(interval);
+            } catch (InterruptedException e) {}
+        }
+        return isRertriable;
+    }
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-http
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Takuma kanari
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-14 00:00:00.000000000 Z
+date: 2015-04-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,8 +58,9 @@ files:
 - src/main/java/org/embulk/input/HttpInputPlugin.java
 - src/main/java/org/embulk/input/ParamsConfig.java
 - src/main/java/org/embulk/input/QueryConfig.java
+- src/main/java/org/embulk/input/RetryHandler.java
 - src/test/java/org/embulk/input/TestHttpInputPlugin.java
-- classpath/embulk-input-http-0.0.4.jar
+- classpath/embulk-input-http-0.0.5.jar
 - classpath/httpclient-4.4.jar
 - classpath/commons-logging-1.2.jar
 - classpath/httpcore-4.4.jar