embulk-input-http 0.0.15 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62da7af46d26103cf5afdcb94b6fb1c807e26c38
-  data.tar.gz: ad8323dab3583326adbcff02408d3eaa3bc8e2a2
+  metadata.gz: 41ace198c24c4c60e9d0266b03480091398fda2e
+  data.tar.gz: a18099e41b799416545996dd6f2518591061ca21
 SHA512:
-  metadata.gz: a8a0e35550e2dd0df465a9b08bd916463676eb818eb1fe00c7279dc799c367e8be5949b04cc513ed4b21fbd01b65fcc7568c3141c27fc5fb58ec556c663e0945
-  data.tar.gz: aa5b119f9a2f29ff4ce01a3b548149ede29de765c8e0431a64862f053646b9bc4f67320299341526de695d56237972b44f53ce710a1349dd6fb642b153a3c4b1
+  metadata.gz: 81e9f5146533fe1944b896ecfe81f4fdc929eda9a16c5917c65cda83c65f0a89b6b2005ee5bc2d19cc04b2b21207810e4f55650ca646a6825868658578dbbd63
+  data.tar.gz: 2004dd879f0518827f4d00ec824bbf26901a469df511443a6b90b48a8102796c65725f193fd36b81f7d874d6a3bb6dc81c19aaf1db91c25e270f1b3e01aceed1

data/README.md

@@ -34,14 +34,15 @@ in:
 - **method**: http method, get is used by default (optional)
 - **user_agent**: the usrr agent to specify request header (optional)
 - **request_headers**: the extra request headers as key-value (optional)
-- **charset**: charset to specify request header (optional, utf-8 is used by default)
+- **charset**: charset to specify request header (optional, default: utf8)
 - **basic_auth**: username/password for basic authentication (optional)
-- **open_timeout**: timeout msec to open connection (optional, 2000 is used by default)
-- **read_timeout**: timeout msec to read content via http (optional, 10000 is used by default)
-- **max_retries**: max number of retry request if failed (optional, 5 is used by default)
-- **retry_interval**: interval msec to retry max (optional, 10000 is used by default)
-- **request_interval**: wait msec before each requests (optional, 0 is used by default)
-- **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, no is used by default)
+- **open_timeout**: timeout msec to open connection (optional, default: 2000)
+- **read_timeout**: timeout msec to read content via http (optional, default: 10000)
+- **max_retries**: max number of retry request if failed (optional, default: 5)
+- **retry_interval**: interval msec to retry max (optional, default: 10000)
+- **request_interval**: wait msec before each requests (optional, default: 0)
+- **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, default: no)
+- **direct\_input**: If false, dumps content to temp file first, to avoid read timeout due to process large data while dowoloading from remote (optional, default: true)
 
 ### Brace expansion style in 'params'
 
@@ -74,8 +75,8 @@ You can specify username/password for basic authentication.
 
 ```yaml
 basic_auth:
- - user: MyUser
- - password: MyPassword
+ user: MyUser
+ password: MyPassword
 ```
 
 ### Paging by 'pager'

data/build.gradle

@@ -13,15 +13,16 @@ configurations {
     provided
 }
 
-version = "0.0.15"
+version = "0.0.17"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.8"
-    compile  "org.apache.httpcomponents:httpclient:4.5"
     provided "org.embulk:embulk-core:0.8.8"
+    compile  "org.apache.httpcomponents:httpclient:4.5"
+    compile  "commons-io:commons-io:2.5"
     testCompile "junit:junit:4.+"
 }
 
@@ -49,13 +50,24 @@ task checkstyle(type: Checkstyle) {
     source = sourceSets.main.allJava + sourceSets.test.allJava
 }
 
-task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "build/gemspec"
+    script "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
-task gemspec << { file("build/gemspec").write($/
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) << {
+    println "> Build succeeded."
+    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+}
+
+
+task gemspec << { file("${project.name}.gemspec").write($/
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"

data/embulk-input-http.gemspec

@@ -0,0 +1,17 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-http"
+  spec.version       = "0.0.17"
+  spec.authors       = ["Takuma kanari"]
+  spec.email         = ["chemtrails.t@gmail.com"]
+  spec.summary       = %q{Embulk plugin for http input}
+  spec.description   = %q{Fetch data via http}
+  spec.homepage      = "https://github.com/takumakanari/embulk-input-http"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/example/json-example.yml

@@ -14,6 +14,7 @@ in:
   interval_includes_response_time: 1
   request_headers: {X-Some-Key1: some-value1, X-Some-key2: some-value2}
   pager: {from_param: from, to_param: to, pages: 10, step: 100}
+#  input_direct: false
   parser:
     type: none
 

data/src/main/java/org/embulk/input/http/{BasicAuthConfig.java → BasicAuthOption.java}

@@ -4,36 +4,40 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Objects;
 
-public class BasicAuthConfig {
-
+public class BasicAuthOption
+{
     private final String user;
     private final String password;
 
     @JsonCreator
-    public BasicAuthConfig(
-            @JsonProperty("user") String user,
-            @JsonProperty("password") String password) {
+    public BasicAuthOption(@JsonProperty("user") String user,
+            @JsonProperty("password") String password)
+    {
         this.user = user;
         this.password = password;
     }
 
     @JsonProperty("user")
-    public String getUser() {
+    public String getUser()
+    {
         return user;
     }
 
     @JsonProperty("password")
-    public String getPassword() {
+    public String getPassword()
+    {
         return password;
     }
 
     @Override
-    public int hashCode() {
+    public int hashCode()
+    {
         return Objects.hashCode(user, password);
     }
 
     @Override
-    public String toString() {
-        return String.format("BasicAuthConfig[%s, %s]", getUser(), getPassword());
+    public String toString()
+    {
+        return String.format("BasicAuthOption[%s, %s]", getUser(), getPassword());
     }
 }

data/src/main/java/org/embulk/input/http/HttpFileInputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.input.http;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
+import org.apache.commons.io.IOUtils;
 import org.apache.http.Header;
 import org.apache.http.NameValuePair;
 import org.apache.http.auth.AuthScope;
@@ -19,7 +20,14 @@ import org.apache.http.impl.client.BasicCredentialsProvider;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicHeader;
 import org.apache.http.message.BasicNameValuePair;
-import org.embulk.config.*;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -28,108 +36,124 @@ import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.URISyntaxException;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 
-public class HttpFileInputPlugin implements FileInputPlugin {
+import static java.lang.String.format;
 
+public class HttpFileInputPlugin implements FileInputPlugin
+{
     private final Logger logger = Exec.getLogger(getClass());
 
-    public interface PluginTask extends Task {
+    public interface PluginTask extends Task
+    {
         @Config("url")
-        public String getUrl();
+        String getUrl();
 
         @Config("charset")
         @ConfigDefault("\"utf-8\"")
-        public String getCharset();
+        String getCharset();
 
         @Config("method")
         @ConfigDefault("\"get\"")
-        public String getMethod();
+        String getMethod();
 
         @Config("user_agent")
         @ConfigDefault("\"Embulk::Input::HttpFileInputPlugin\"")
-        public String getUserAgent();
+        String getUserAgent();
 
         @Config("open_timeout")
         @ConfigDefault("2000")
-        public int getOpenTimeout();
+        int getOpenTimeout();
 
         @Config("read_timeout")
         @ConfigDefault("10000")
-        public int getReadTimeout();
+        int getReadTimeout();
 
         @Config("max_retries")
         @ConfigDefault("5")
-        public int getMaxRetries();
+        int getMaxRetries();
 
         @Config("retry_interval")
         @ConfigDefault("10000")
-        public int getRetryInterval();
+        int getRetryInterval();
 
         @Config("request_interval")
         @ConfigDefault("0")
-        public int getRequestInterval();
+        int getRequestInterval();
 
-        public void setRequestInterval(int requestInterval);
+        void setRequestInterval(int requestInterval);
 
         @Config("interval_includes_response_time")
         @ConfigDefault("null")
-        public boolean getIntervalIncludesResponseTime();
+        boolean getIntervalIncludesResponseTime();
+
+        @Config("input_direct")
+        @ConfigDefault("true")
+        boolean getInputDirect();
 
         @Config("params")
         @ConfigDefault("null")
-        public Optional<ParamsConfig> getParams();
+        Optional<ParamsOption> getParams();
 
         @Config("basic_auth")
         @ConfigDefault("null")
-        public Optional<BasicAuthConfig> getBasicAuth();
+        Optional<BasicAuthOption> getBasicAuth();
 
         @Config("pager")
         @ConfigDefault("null")
-        public Optional<PagerConfig> getPager();
+        Optional<PagerOption> getPager();
 
         @Config("request_headers")
         @ConfigDefault("{}")
-        public Map<String, String> getRequestHeaders();
+        Map<String, String> getRequestHeaders();
 
         @ConfigInject
-        public BufferAllocator getBufferAllocator();
+        BufferAllocator getBufferAllocator();
 
-        public List<List<QueryConfig.Query>> getQueries();
+        List<List<QueryOption.Query>> getQueries();
 
-        public void setQueries(List<List<QueryConfig.Query>> queries);
+        void setQueries(List<List<QueryOption.Query>> queries);
 
-        public HttpMethod getHttpMethod();
+        HttpMethod getHttpMethod();
 
-        public void setHttpMethod(HttpMethod httpMethod);
+        void setHttpMethod(HttpMethod httpMethod);
     }
 
-    public enum HttpMethod {
+    public enum HttpMethod
+    {
         POST,
         GET
     }
 
     @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
         PluginTask task = config.loadConfig(PluginTask.class);
 
         final int tasks;
         if (task.getParams().isPresent()) {
-            List<List<QueryConfig.Query>> queries = task.getParams().get().generateQueries(task.getPager());
+            List<List<QueryOption.Query>> queries = task.getParams().get().generateQueries(task.getPager());
             task.setQueries(queries);
             tasks = queries.size();
-        } else if (task.getPager().isPresent()) {
-            List<List<QueryConfig.Query>> queries = task.getPager().get().expand();
+        }
+        else if (task.getPager().isPresent()) {
+            List<List<QueryOption.Query>> queries = task.getPager().get().expand();
             task.setQueries(queries);
             tasks = queries.size();
-        } else {
-            task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
+        }
+        else {
+            task.setQueries(Lists.<List<QueryOption.Query>>newArrayList());
             task.setRequestInterval(0);
             tasks = 1;
         }
@@ -140,25 +164,27 @@ public class HttpFileInputPlugin implements FileInputPlugin {
     }
 
     @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control) {
+    public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control)
+    {
         control.run(taskSource, taskCount);
         return Exec.newConfigDiff();
     }
 
     @Override
-    public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
+    public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports)
+    {
     }
 
     @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
         HttpRequestBase request;
         try {
             request = makeRequest(task, taskIndex);
-        } catch (URISyntaxException | UnsupportedEncodingException e) {
+        }
+        catch (URISyntaxException | UnsupportedEncodingException e) {
             throw Throwables.propagate(e);
         }
 
@@ -173,8 +199,7 @@ public class HttpFileInputPlugin implements FileInputPlugin {
 
         HttpClient client = builder.build();
 
-        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
-                request.getURI().toString()));
+        logger.info(format(Locale.ENGLISH, "%s \"%s\"", task.getMethod().toUpperCase(), request.getURI().toString()));
 
         RetryableHandler retryable = new RetryableHandler(client, request);
         long startTimeMills = System.currentTimeMillis();
@@ -184,33 +209,57 @@ public class HttpFileInputPlugin implements FileInputPlugin {
                     withInitialRetryWait(task.getRetryInterval()).
                     withMaxRetryWait(30 * 60 * 1000).
                     runInterruptible(retryable);
+
             InputStream stream = retryable.getResponse().getEntity().getContent();
+            if (!task.getInputDirect()) {
+                stream = copyToFile(stream);
+            }
+
             PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
             stream = null;
             return input;
-        } catch (Exception e) {
+        }
+        catch (Exception e) {
             throw Throwables.propagate(e);
         }
     }
 
-    private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
+    private InputStream copyToFile(InputStream input)
+            throws IOException
+    {
+        File tmpfile = Files.createTempFile("embulk-input-http.", ".tmp").toFile();
+        tmpfile.deleteOnExit();
+
+        try (FileOutputStream output = new FileOutputStream(tmpfile)) {
+            logger.info(format(Locale.ENGLISH, "Writing response to %s", tmpfile));
+            IOUtils.copy(input, output);
+        } finally {
+            input.close();
+        }
+
+        return new FileInputStream(tmpfile);
+    }
+
+    private CredentialsProvider makeCredentialsProvider(BasicAuthOption basicAuth, HttpRequestBase request)
+    {
         final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
-        final AuthScope authScope = new AuthScope(scopeRequest.getURI().getHost(),
-                scopeRequest.getURI().getPort());
+        final AuthScope authScope = new AuthScope(request.getURI().getHost(),
+                request.getURI().getPort());
         credentialsProvider.setCredentials(authScope,
-                new UsernamePasswordCredentials(config.getUser(), config.getPassword()));
+                new UsernamePasswordCredentials(basicAuth.getUser(), basicAuth.getPassword()));
         return credentialsProvider;
     }
 
     private HttpRequestBase makeRequest(PluginTask task, int taskIndex)
-            throws URISyntaxException, UnsupportedEncodingException {
-        final List<QueryConfig.Query> queries = (task.getQueries().isEmpty()) ?
+            throws URISyntaxException, UnsupportedEncodingException
+    {
+        final List<QueryOption.Query> queries = (task.getQueries().isEmpty()) ?
                 null : task.getQueries().get(taskIndex);
         if (task.getHttpMethod() == HttpMethod.GET) {
             HttpGet request = new HttpGet(task.getUrl());
             if (queries != null) {
                 URIBuilder builder = new URIBuilder(request.getURI());
-                for (QueryConfig.Query q : queries) {
+                for (QueryOption.Query q : queries) {
                     for (String v : q.getValues()) {
                         builder.addParameter(q.getName(), v);
                     }
@@ -218,11 +267,12 @@ public class HttpFileInputPlugin implements FileInputPlugin {
                 request.setURI(builder.build());
             }
             return request;
-        } else if (task.getHttpMethod() == HttpMethod.POST) {
+        }
+        else if (task.getHttpMethod() == HttpMethod.POST) {
             HttpPost request = new HttpPost(task.getUrl());
             if (queries != null) {
                 List<NameValuePair> pairs = new ArrayList<>();
-                for (QueryConfig.Query q : queries) {
+                for (QueryOption.Query q : queries) {
                     for (String v : q.getValues()) {
                         pairs.add(new BasicNameValuePair(q.getName(), v));
                     }
@@ -234,7 +284,8 @@ public class HttpFileInputPlugin implements FileInputPlugin {
         throw new IllegalArgumentException(String.format("Unsupported http method %s", task.getMethod()));
     }
 
-    private List<Header> makeHeaders(PluginTask task) {
+    private List<Header> makeHeaders(PluginTask task)
+    {
         List<Header> headers = new ArrayList<>();
         headers.add(new BasicHeader("Accept", "*/*"));
         headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
@@ -247,7 +298,8 @@ public class HttpFileInputPlugin implements FileInputPlugin {
         return headers;
     }
 
-    private RequestConfig makeRequestConfig(PluginTask task) {
+    private RequestConfig makeRequestConfig(PluginTask task)
+    {
         return RequestConfig.custom()
                 .setCircularRedirectsAllowed(true)
                 .setMaxRedirects(10)
@@ -258,34 +310,39 @@ public class HttpFileInputPlugin implements FileInputPlugin {
     }
 
     public static class PluginFileInput extends InputStreamFileInput
-            implements TransactionalFileInput {
-
+            implements TransactionalFileInput
+    {
         private final Logger logger = Exec.getLogger(getClass());
 
         private final long startTimeMills;
         private final PluginTask task;
 
-        public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
+        public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills)
+        {
             super(task.getBufferAllocator(), new SingleFileProvider(stream));
             this.startTimeMills = startTimeMills;
             this.task = task;
         }
 
-        public TaskReport commit() {
+        public TaskReport commit()
+        {
             return Exec.newTaskReport();
         }
 
         @Override
-        public void close() {
+        public void close()
+        {
             super.close();
             handleInterval();
         }
 
         @Override
-        public void abort() {
+        public void abort()
+        {
         }
 
-        protected void handleInterval() {
+        protected void handleInterval()
+        {
             if (task.getRequestInterval() <= 0) {
                 return;
             }
@@ -297,24 +354,27 @@ public class HttpFileInputPlugin implements FileInputPlugin {
                 logger.info(String.format("waiting %d msec ...", interval));
                 try {
                     Thread.sleep(interval);
-                } catch (InterruptedException e) {
-                    Throwables.propagate(e);
+                }
+                catch (InterruptedException e) {
+                    throw Throwables.propagate(e);
                 }
             }
         }
 
         private static class SingleFileProvider
-                implements InputStreamFileInput.Provider {
-
+                implements InputStreamFileInput.Provider
+        {
             private final InputStream stream;
             private boolean opened = false;
 
-            public SingleFileProvider(InputStream stream) {
+            public SingleFileProvider(InputStream stream)
+            {
                 this.stream = stream;
             }
 
             @Override
-            public InputStream openNext() throws IOException {
+            public InputStream openNext() throws IOException
+            {
                 if (opened) {
                     return null;
                 }
@@ -323,7 +383,8 @@ public class HttpFileInputPlugin implements FileInputPlugin {
             }
 
             @Override
-            public void close() throws IOException {
+            public void close() throws IOException
+            {
                 if (!opened) {
                     stream.close();
                 }