embulk-input-http 0.0.3 → 0.0.4

data/lib/embulk/input/http.rb

@@ -1,196 +1,3 @@
-require "net/http"
-require "uri"
-require "bracecomp"
-
-module Embulk
-  module Input
-
-    class HttpInputPlugin < InputPlugin
-      Plugin.register_input("http", self)
-
-      def self.transaction(config, &control)
-        task = {
-          :url => config.param("url", :string),
-          :method => config.param("method", :string, default: "get"),
-          :schema => config.param("schema", :array),
-          :iterate => config.param("iterate", :hash),
-          :open_timeout => config.param("open_timeout", :float, default: 2.0),
-          :read_timeout => config.param("read_timeout", :float, default: 10.0),
-          :done => config.param("done", :array, default: [])
-        }
-        params = config.param("params", :array, default: [])
-        params_unexpand, params_expand = configure_queries(params)
-
-        data_type = task[:iterate]["type"]
-        unless ["json", "xml"].include?(data_type)
-          raise "Unknown data_type #{data_type}, only supported for json or xml"
-        end
-
-        columns = task[:schema].each_with_index.map do |c, i|
-          Column.new(i, c["name"], c["type"].to_sym)
-        end
-
-        task[:params] = params_unexpand
-        task[:params_expand] = params_expand - task[:done]
-        num_of_threads = task[:params_expand].empty? ? 1 : task[:params_expand].size
-
-        report = yield(task, columns, num_of_threads)
-        {"done" => report.map{|r| r["done"]}.compact}
-      end
-
-      def self.configure_queries(params)
-        base = params.select{|p| !p["expand"]}.map do |p|
-          [p["name"], p["value"]]
-        end
-        expands = params.select{|p| p["expand"] }.map do |p|
-          p["value"].expand.map do |v|
-            [p["name"], v]
-          end
-        end
-        if expands.size > 0
-            dest = expands.first.product(*(expands.slice(1, expands.size - 1)))
-            dest.sort!{|a, b| "#{a[0]}=#{a[1]}" <=> "#{b[0]}=#{b[1]}"}
-        else
-            dest = []
-        end
-        [base, dest]
-      end
-
-      def run
-        schema = @task["schema"]
-        iterate = @task["iterate"]
-        url = @task["url"]
-        method = @task["method"]
-
-        params_expand = @task["params_expand"][@index] || []
-        query = URI.encode_www_form(@task["params"] + params_expand)
-        puts "#{@index}: #{method.upcase} #{url}?#{query}"
-
-        data = fetch(url, method, query).body
-        data_type = iterate["type"]
-
-        case data_type
-          when "json"
-            iter = IterJson.new(schema, data, iterate)
-          when "xml"
-            iter = IterXML.new(schema, data, iterate)
-          else
-            raise "Unsupported data_type #{data_type}"
-        end
-
-        iter.each do |record|
-          @page_builder.add(record)
-        end
-        @page_builder.finish
-
-        {:done => params_expand}
-      end
-
-      private
-
-      def fetch(url, method, query)
-        uri = URI.parse(url)
-
-        res = Net::HTTP.start(uri.host, uri.port) do |client|
-          client.open_timeout = @task["open_timeout"]
-          client.read_timeout = @task["read_timeout"]
-          case method.downcase
-            when "get"
-              client.get([uri.path, query].join("?"))
-            when "post"
-              client.post(uri.path, query)
-            else
-              raise "Unsupported method #{method}"
-          end
-        end
-
-        case res
-          when Net::HTTPSuccess
-            res
-          else
-            raise "Request is not successful, code=#{res.code}, value=#{res.body}"
-        end
-      end
-
-      class Iter
-        def initialize(schema, data, config)
-          @schema = schema
-          @data = data
-          @config = config
-        end
-
-        def each
-          raise NotImplementedError.new("each")
-        end
-
-        private
-
-        def make_record(e)
-          @schema.map do |c|
-            name = c["name"]
-            path = c["path"]
-            val = path.nil? ? e[name] : find_by_path(e, path)
-
-            v = val.nil? ? "" : val
-            type = c["type"]
-            case type
-              when "string"
-                v
-              when "long"
-                v.to_i
-              when "double"
-                v.to_f
-              when "boolean"
-                ["yes", "true", "1"].include?(v)
-              when "timestamp"
-                v.empty? ? nil : Time.strptime(v, c["format"])
-              else
-                raise "Unsupported type #{type}"
-            end
-          end
-        end
-
-        def find_by_path(e, path)
-          raise NotImplementedError.new("Find by path is unsupported")
-        end
-      end
-
-      class IterXML < Iter
-        def initialize(schema, data, config)
-          require "rexml/document"
-          super
-          @doc = REXML::Document.new(@data)
-        end
-
-        def each
-          @doc.elements.each(@config["path"]) do |e|
-            dest = {}
-            e.elements.each do |d|
-              dest[d.name] = d.text
-            end
-            yield make_record(dest)
-          end
-        end
-      end
-
-      class IterJson < Iter
-        def initialize(schema, data, config)
-          require "jsonpath"
-          super
-          @jsonpath = JsonPath.new(@config["path"])
-        end
-
-        def each
-          @jsonpath.on(@data).flatten.each do |e|
-            yield make_record(e)
-          end
-        end
-
-        def find_by_path(e, path)
-          JsonPath.on(e, path).first
-        end
-      end
-
-    end
-  end
-end
+Embulk::JavaPlugin.register_input(
+  "http", "org.embulk.input.HttpInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -0,0 +1,258 @@
+package org.embulk.input;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.apache.http.Header;
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.util.EntityUtils;
+import org.embulk.config.*;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class HttpInputPlugin implements FileInputPlugin {
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends Task {
+        @Config("url")
+        public String getUrl();
+
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public String getCharset();
+
+        @Config("method")
+        @ConfigDefault("\"get\"")
+        public String getMethod();
+
+        @Config("user_agent")
+        @ConfigDefault("\"Embulk::Input::HttpInputPlugin\"")
+        public String getUserAgent();
+
+        @Config("open_timeout")
+        @ConfigDefault("2000")
+        public int getOpenTimeout();
+
+        @Config("read_timeout")
+        @ConfigDefault("10000")
+        public int getReadTimeout();
+
+        @Config("params")
+        @ConfigDefault("null")
+        public Optional<ParamsConfig> getParams();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+
+        public List<ParamsConfig> getQueries();
+        public void setQueries(List<ParamsConfig> queries);
+
+        public HttpMethod getHttpMethod();
+        public void setHttpMethod(HttpMethod httpMethod);
+    }
+
+    public enum HttpMethod {
+        POST,
+        GET
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        int numOfThreads = 1;
+        if (task.getParams().isPresent()) {
+            List<ParamsConfig> expandedQueries = task.getParams().get().expandQueries();
+            task.setQueries(expandedQueries);
+            numOfThreads = expandedQueries.size();
+        } else {
+            task.setQueries(new ArrayList<ParamsConfig>());
+        }
+
+        switch (task.getMethod().toUpperCase()) {
+            case "GET":
+                task.setHttpMethod(HttpMethod.GET);
+                break;
+            case "POST":
+                task.setHttpMethod(HttpMethod.POST);
+                break;
+            default:
+                throw new ConfigException(String.format("Unsupported http method %s", task.getMethod()));
+        }
+
+        return resume(task.dump(), numOfThreads, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+                             int taskCount,
+                             FileInputPlugin.Control control) {
+        control.run(taskSource, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+                        int taskCount,
+                        List<CommitReport> successCommitReports) {
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        HttpRequestBase request;
+        try {
+            request = makeRequest(task, taskIndex);
+        } catch (URISyntaxException | UnsupportedEncodingException e) {
+            throw Throwables.propagate(e);
+        }
+        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
+                request.getURI().toString()));
+
+        HttpClient client = HttpClientBuilder.create()
+                .setDefaultRequestConfig(makeRequestConfig(task))
+                .setDefaultHeaders(makeHeaders(task))
+                .build();
+        try {
+            HttpResponse response = client.execute(request);
+            statusIsOkOrThrow(response);
+            InputStream stream = response.getEntity().getContent();
+            PluginFileInput input = new PluginFileInput(task, stream);
+            stream = null;
+            return input;
+        } catch (IOException | HttpException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+
+    private HttpRequestBase makeRequest(PluginTask task, int taskIndex)
+            throws URISyntaxException, UnsupportedEncodingException {
+        final ParamsConfig paramsConfig = (task.getQueries().isEmpty()) ?
+                null : task.getQueries().get(taskIndex);
+        if (task.getHttpMethod() == HttpMethod.GET) {
+            HttpGet request = new HttpGet(task.getUrl());
+            if (paramsConfig != null) {
+                URIBuilder builder = new URIBuilder(request.getURI());
+                for (QueryConfig p : paramsConfig.getQueries()) {
+                    builder.addParameter(p.getName(), p.getValue());
+                }
+                request.setURI(builder.build());
+            }
+            return request;
+        } else if (task.getHttpMethod() == HttpMethod.POST) {
+            HttpPost request = new HttpPost(task.getUrl());
+            if (paramsConfig != null) {
+                List<NameValuePair> pairs = new ArrayList<>();
+                for (QueryConfig p : paramsConfig.getQueries()) {
+                    pairs.add(new BasicNameValuePair(p.getName(), p.getValue()));
+                }
+                request.setEntity(new UrlEncodedFormEntity(pairs));
+            }
+            return request;
+        }
+        throw new IllegalArgumentException(String.format("Unsupported http method %s", task.getMethod()));
+    }
+
+    private List<Header> makeHeaders(PluginTask task) {
+        List<Header> headers = new ArrayList<>();
+        headers.add(new BasicHeader("Accept", "*/*"));
+        headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
+        headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
+        headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
+        headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
+        return headers;
+    }
+
+    private RequestConfig makeRequestConfig(PluginTask task) {
+        return RequestConfig.custom()
+                .setCircularRedirectsAllowed(true)
+                .setMaxRedirects(10)
+                .setRedirectsEnabled(true)
+                .setConnectTimeout(task.getOpenTimeout())
+                .setSocketTimeout(task.getReadTimeout())
+                .build();
+    }
+
+    private void statusIsOkOrThrow(HttpResponse response)
+            throws HttpException, IOException {
+        int code = response.getStatusLine().getStatusCode();
+        switch (response.getStatusLine().getStatusCode()) {
+            case 200:
+                return;
+            default:
+                throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
+                        code, EntityUtils.toString(response.getEntity())));
+        }
+    }
+
+    public static class PluginFileInput extends InputStreamFileInput
+            implements TransactionalFileInput {
+
+        private static class SingleFileProvider
+                implements InputStreamFileInput.Provider {
+
+            private InputStream stream;
+            private boolean opened = false;
+
+            public SingleFileProvider(InputStream stream) {
+                this.stream = stream;
+            }
+
+            @Override
+            public InputStream openNext() throws IOException {
+                if (opened) {
+                    return null;
+                }
+                opened = true;
+                return stream;
+            }
+
+            @Override
+            public void close() throws IOException {
+                if (!opened) {
+                    stream.close();
+                }
+            }
+        }
+
+        public PluginFileInput(PluginTask task, InputStream stream) {
+            super(task.getBufferAllocator(), new SingleFileProvider(stream));
+        }
+
+        public void abort() {
+        }
+
+        public CommitReport commit() {
+            return Exec.newCommitReport();
+        }
+
+        @Override
+        public void close() {
+        }
+    }
+
+}

data/src/main/java/org/embulk/input/ParamsConfig.java

@@ -0,0 +1,66 @@
+package org.embulk.input;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Objects;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ParamsConfig {
+
+    private final List<QueryConfig> queries;
+
+    @JsonCreator
+    public ParamsConfig(List<QueryConfig> queries) {
+        this.queries = queries;
+    }
+
+    @JsonValue
+    public List<QueryConfig> getQueries() {
+        return queries;
+    }
+
+    public List<ParamsConfig> expandQueries() {
+        List<List<QueryConfig>> base = new ArrayList<>(queries.size());
+        for (QueryConfig p : queries) {
+            base.add(p.expand());
+        }
+
+        int productSize = 1;
+        int baseSize = base.size();
+        for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++);
+
+        List<ParamsConfig> expands = new ArrayList<>(productSize);
+        for(int i = 0; i < productSize; i++) {
+            int j = 1;
+            List<QueryConfig> query = new ArrayList<>();
+            for(List<QueryConfig> list : base) {
+                QueryConfig pc = list.get((i / j) % list.size());
+                query.add(pc);
+                j *= list.size();
+            }
+            expands.add(new ParamsConfig(query));
+        }
+
+        return expands;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ParamsConfig)) {
+            return false;
+        }
+        ParamsConfig other = (ParamsConfig) obj;
+        return Objects.equal(queries, other.queries);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(queries);
+    }
+
+}

data/src/main/java/org/embulk/input/QueryConfig.java

@@ -0,0 +1,125 @@
+package org.embulk.input;
+
+import com.google.common.base.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class QueryConfig {
+
+    private final String name;
+    private final String value;
+    private final boolean expand;
+
+    @JsonCreator
+    public QueryConfig(
+            @JsonProperty("name") String name,
+            @JsonProperty("value") String value,
+            @JsonProperty("expand") boolean expand) {
+        this.name = name;
+        this.value = value;
+        this.expand = expand;
+    }
+
+    public List<QueryConfig> expand() {
+        List<QueryConfig> dest;
+        if (!expand) {
+            dest = new ArrayList<>(1);
+            dest.add(this);
+        } else {
+            List<String> expanded = BraceExpansion.expand(value);
+            dest = new ArrayList<>(expanded.size());
+            for(String s : expanded) {
+                dest.add(new QueryConfig(name, s, false));
+            }
+        }
+        return dest;
+    }
+
+    @JsonProperty("name")
+    public String getName() {
+        return name;
+    }
+
+    @JsonProperty("value")
+    public String getValue() {
+        return value;
+    }
+
+    @JsonProperty("expand")
+    public boolean isExpand() {
+        return expand;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof QueryConfig)) {
+            return false;
+        }
+        QueryConfig other = (QueryConfig) obj;
+        return Objects.equal(this.name, other.name) &&
+                Objects.equal(value, other.value) &&
+                Objects.equal(expand, other.expand);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(name, value, expand);
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ParameterConfig[%s, %s, %s]",
+                getName(), getValue(), isExpand());
+    }
+
+    private static class BraceExpansion {
+
+        public static List<String> expand(String s) {
+            return expandRecursive("", s, "", new ArrayList<String>());
+        }
+
+        private static List<String> expandRecursive(String prefix, String s,
+                                                    String suffix, List<String> dest) {
+            // I used the code below as reference.
+            //  http://rosettacode.org/wiki/Brace_expansion#Java
+            int i1 = -1, i2 = 0;
+            String noEscape = s.replaceAll("([\\\\]{2}|[\\\\][,}{])", "  ");
+            StringBuilder sb = null;
+
+            outer:
+            while ((i1 = noEscape.indexOf('{', i1 + 1)) != -1) {
+                i2 = i1 + 1;
+                sb = new StringBuilder(s);
+                for (int depth = 1; i2 < s.length() && depth > 0; i2++) {
+                    char c = noEscape.charAt(i2);
+                    depth = (c == '{') ? ++depth : depth;
+                    depth = (c == '}') ? --depth : depth;
+                    if (c == ',' && depth == 1) {
+                        sb.setCharAt(i2, '\u0000');
+                    } else if (c == '}' && depth == 0 && sb.indexOf("\u0000") != -1) {
+                        break outer;
+                    }
+                }
+            }
+
+            if (i1 == -1) {
+                if (suffix.length() > 0) {
+                    expandRecursive(prefix + s, suffix, "", dest);
+                } else {
+                    dest.add(String.format("%s%s%s", prefix, s, suffix));
+                }
+            } else {
+                for (String m : sb.substring(i1 + 1, i2).split("\u0000", -1)) {
+                    expandRecursive(prefix + s.substring(0, i1), m, s.substring(i2 + 1) + suffix, dest);
+                }
+            }
+            return dest;
+        }
+    }
+}

data/src/test/java/org/embulk/input/TestHttpInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input;
+
+public class TestHttpInputPlugin
+{
+}