embulk-input-http 0.0.14 → 0.0.15

data/src/main/java/org/embulk/input/http/PagerConfig.java

@@ -0,0 +1,86 @@
+package org.embulk.input.http;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PagerConfig {
+
+    private final String fromParam;
+    private final Optional<String> toParam;
+    private final int start;
+    private final int pages;
+    private final int step;
+
+    @JsonCreator
+    public PagerConfig(@JsonProperty("from_param") String fromParam,
+            @JsonProperty("to_param") Optional<String> toParam,
+            @JsonProperty("start") Optional<Integer> start,
+            @JsonProperty("pages") int pages,
+            @JsonProperty("step") Optional<Integer> step) {
+        this.fromParam = fromParam;
+        this.toParam = toParam;
+        this.start = start.or(0);
+        this.pages = pages;
+        this.step = step.or(1);
+    }
+
+    public List<List<QueryConfig.Query>> expand() {
+        List<List<QueryConfig.Query>> queries = new ArrayList<>();
+        int p = 1;
+        int index = start;
+        while (p <= pages) {
+            List<QueryConfig.Query> one = new ArrayList<>();
+            one.add(new QueryConfig.Query(fromParam, Integer.toString(index)));
+            if (toParam.isPresent()) {
+                int t = index + step - 1;
+                one.add(new QueryConfig.Query(toParam.get(), Integer.toString(t)));
+                index = t + 1;
+            } else {
+                index += step;
+            }
+            queries.add(one);
+            p++;
+        }
+        return queries;
+    }
+
+    @JsonProperty("from_param")
+    public String getFromParam() {
+        return fromParam;
+    }
+
+    @JsonProperty("to_param")
+    public Optional<String> getToParam() {
+        return toParam;
+    }
+
+    @JsonProperty("start")
+    public int getStart() {
+        return start;
+    }
+
+    @JsonProperty("pages")
+    public int getPages() {
+        return pages;
+    }
+
+    @JsonProperty("step")
+    public int getStep() {
+        return step;
+    }
+
+    @Override
+    public String toString() {
+        return "PagerConfig{" +
+                "fromParam='" + fromParam + '\'' +
+                ", toParam=" + toParam +
+                ", start=" + start +
+                ", pages=" + pages +
+                ", step=" + step +
+                '}';
+    }
+}

data/src/main/java/org/embulk/input/http/ParamsConfig.java

@@ -0,0 +1,82 @@
+package org.embulk.input.http;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ParamsConfig {
+
+    private final List<QueryConfig> queries;
+
+    @JsonCreator
+    public ParamsConfig(List<QueryConfig> queries) {
+        this.queries = queries;
+    }
+
+    @JsonValue
+    public List<QueryConfig> getQueries() {
+        return queries;
+    }
+
+    public List<List<QueryConfig.Query>> generateQueries(Optional<PagerConfig> pagerConfig) {
+        List<List<QueryConfig.Query>> base = new ArrayList<>(queries.size());
+        for (QueryConfig p : queries) {
+            base.add(p.expand());
+        }
+
+        int productSize = 1;
+        int baseSize = base.size();
+        for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++);
+
+        List<List<QueryConfig.Query>> expands = new ArrayList<>(productSize);
+        for (int i = 0; i < productSize; i++) {
+            int j = 1;
+            List<QueryConfig.Query> one = new ArrayList<>();
+            for (List<QueryConfig.Query> list : base) {
+                QueryConfig.Query pc = list.get((i / j) % list.size());
+                one.add(pc);
+                j *= list.size();
+            }
+            if (pagerConfig.isPresent()) {
+                for (List<QueryConfig.Query> q : pagerConfig.get().expand()) {
+                    expands.add(copyAndConcat(one, q));
+                }
+            } else {
+                expands.add(one);
+            }
+        }
+
+        return expands;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ParamsConfig)) {
+            return false;
+        }
+        ParamsConfig other = (ParamsConfig) obj;
+        return Objects.equal(queries, other.queries);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(queries);
+    }
+
+    private List<QueryConfig.Query> copyAndConcat(List<QueryConfig.Query>... srcs) {
+        List<QueryConfig.Query> dest = new ArrayList<>();
+        for (List<QueryConfig.Query> src : srcs) {
+            for (QueryConfig.Query q : src) {
+                dest.add(q.copy());
+            }
+        }
+        return dest;
+    }
+}

data/src/main/java/org/embulk/input/http/QueryConfig.java

@@ -0,0 +1,171 @@
+package org.embulk.input.http;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Objects;
+import com.google.common.base.Optional;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class QueryConfig {
+
+    private final String name;
+    private final Optional<String> value;
+    private final Optional<List<String>> values;
+    private final boolean expand;
+
+    @JsonCreator
+    public QueryConfig(
+            @JsonProperty("name") String name,
+            @JsonProperty("value") Optional<String> value,
+            @JsonProperty("values") Optional<List<String>> values,
+            @JsonProperty("expand") boolean expand) {
+        this.name = name;
+        this.value = value;
+        this.values = values;
+        this.expand = expand;
+    }
+
+    public List<Query> expand() {
+        List<Query> dest;
+        if (value.isPresent()) {
+            if (expand) {
+                List<String> expanded = BraceExpansion.expand(value.get());
+                dest = new ArrayList<>(expanded.size());
+                for (String s : expanded) {
+                    dest.add(new Query(name, s));
+                }
+            } else {
+                dest = new ArrayList<>(1);
+                dest.add(new Query(name, value.get()));
+            }
+        } else if (values.isPresent()) {
+            if (expand) {
+                dest = new ArrayList<>(values.get().size());
+                for (String s : values.get()) {
+                    dest.add(new Query(name, s));
+                }
+            } else {
+                dest = new ArrayList<>(1);
+                final String[] valueArr = values.get().toArray(new String[values.get().size()]);
+                dest.add(new Query(name, valueArr));
+            }
+        } else {
+            throw new IllegalArgumentException("value or values must be specified to 'params'");
+        }
+        return dest;
+    }
+
+    @JsonProperty("name")
+    public String getName() {
+        return name;
+    }
+
+    @JsonProperty("value")
+    public Optional<String> getValue() {
+        return value;
+    }
+
+    @JsonProperty("expand")
+    public boolean isExpand() {
+        return expand;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof QueryConfig)) {
+            return false;
+        }
+        QueryConfig other = (QueryConfig) obj;
+        return Objects.equal(this.name, other.name) &&
+                Objects.equal(value, other.value) &&
+                Objects.equal(expand, other.expand);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(name, value, expand);
+    }
+
+    @Override
+    public String toString() {
+        return String.format("ParameterConfig[%s, %s, %s]",
+                getName(), getValue(), isExpand());
+    }
+
+    public static class Query {
+        private final String name;
+        private final String[] values;
+
+        public Query(
+                @JsonProperty("name") String name,
+                @JsonProperty("values") String... values) {
+            this.name = name;
+            this.values = values;
+        }
+
+        public String getName() {
+            return name;
+        }
+
+        public String[] getValues() {
+            return values;
+        }
+
+        public Query copy() {
+            return new Query(this.name, Arrays.copyOf(this.values, this.values.length));
+        }
+    }
+
+    private static class BraceExpansion {
+
+        public static List<String> expand(String s) {
+            return expandRecursive("", s, "", new ArrayList<String>());
+        }
+
+        private static List<String> expandRecursive(String prefix, String s,
+                                                    String suffix, List<String> dest) {
+            // I used the code below as reference.
+            //  http://rosettacode.org/wiki/Brace_expansion#Java
+            int i1 = -1, i2 = 0;
+            String noEscape = s.replaceAll("([\\\\]{2}|[\\\\][,}{])", "  ");
+            StringBuilder sb = null;
+
+            outer:
+            while ((i1 = noEscape.indexOf('{', i1 + 1)) != -1) {
+                i2 = i1 + 1;
+                sb = new StringBuilder(s);
+                for (int depth = 1; i2 < s.length() && depth > 0; i2++) {
+                    char c = noEscape.charAt(i2);
+                    depth = (c == '{') ? ++depth : depth;
+                    depth = (c == '}') ? --depth : depth;
+                    if (c == ',' && depth == 1) {
+                        sb.setCharAt(i2, '\u0000');
+                    } else if (c == '}' && depth == 0 && sb.indexOf("\u0000") != -1) {
+                        break outer;
+                    }
+                }
+            }
+
+            if (i1 == -1) {
+                if (suffix.length() > 0) {
+                    expandRecursive(prefix + s, suffix, "", dest);
+                } else {
+                    final String out = String.format("%s%s%s", prefix, s, suffix).
+                            replaceAll("[\\\\]{2}", "\\").replaceAll("[\\\\]([,}{])", "$1");
+                    dest.add(out);
+                }
+            } else {
+                for (String m : sb.substring(i1 + 1, i2).split("\u0000", -1)) {
+                    expandRecursive(prefix + s.substring(0, i1), m, s.substring(i2 + 1) + suffix, dest);
+                }
+            }
+            return dest;
+        }
+    }
+}

data/src/main/java/org/embulk/input/http/RetryableHandler.java

@@ -0,0 +1,90 @@
+package org.embulk.input.http;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.http.HttpException;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.util.EntityUtils;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.UnknownHostException;
+import java.util.List;
+
+public class RetryableHandler implements RetryExecutor.Retryable {
+
+    protected final Logger logger = Exec.getLogger(getClass());
+
+    private static List<Class<? extends IOException>> NOT_RETRIABLE_CLAASSES;
+
+    private final HttpClient client;
+    private final HttpRequestBase request;
+    private HttpResponse response;
+
+    static {
+        ImmutableList.Builder<Class<? extends IOException>> classes = ImmutableList.builder();
+        classes.add(UnknownHostException.class).
+                add(InterruptedIOException.class).
+                add(SSLException.class);
+        NOT_RETRIABLE_CLAASSES = classes.build();
+    }
+
+    public RetryableHandler(HttpClient client, HttpRequestBase request) {
+        this.client = client;
+        this.request = request;
+    }
+
+    public HttpResponse getResponse() {
+        return response;
+    }
+
+    @Override
+    public Object call() throws Exception {
+        if (response != null) throw new IllegalStateException("response is already set");
+        HttpResponse response = client.execute(request);
+        statusIsOkOrThrow(response);
+        this.response = response;
+        return null;
+    }
+
+    @Override
+    public boolean isRetryableException(Exception exception) {
+        if (NOT_RETRIABLE_CLAASSES.contains(exception.getClass())) {
+            logger.error(String.format("'%s' is not retriable", exception.getClass()));
+            return false;
+        }
+        return true;
+    }
+
+    @Override
+    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            throws RetryExecutor.RetryGiveupException {
+        logger.warn("retrying {}/{} after {} seconds. Message: {}",
+                retryCount, retryLimit, retryWait / 1000,
+                exception.getMessage());
+    }
+
+    @Override
+    public void onGiveup(Exception firstException, Exception lastException)
+            throws RetryExecutor.RetryGiveupException {
+        logger.error("giveup {}", lastException.getMessage());
+    }
+
+    protected void statusIsOkOrThrow(HttpResponse response)
+            throws HttpException, IOException {
+        int code = response.getStatusLine().getStatusCode();
+        switch (response.getStatusLine().getStatusCode()) {
+            case 200:
+                return;
+            default:
+                throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
+                        code, EntityUtils.toString(response.getEntity())));
+        }
+    }
+
+}

data/src/test/java/org/embulk/input/http/TestHttpInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input.http;
+
+public class TestHttpInputPlugin
+{
+}

data/src/test/java/org/embulk/input/http/TestPagerConfig.java

@@ -0,0 +1,78 @@
+package org.embulk.input.http;
+
+import com.google.common.base.Optional;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestPagerConfig {
+
+    @Test
+    public void testExpandFromTo() throws Exception {
+        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), Optional.of(1), 3,
+                Optional.of(2)).expand();
+        assertEquals(dest.size(), 3);
+
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "from");
+        assertEquals(dest.get(0).get(0).getValues()[0], "1");
+        assertEquals(dest.get(0).get(1).getName(), "to");
+        assertEquals(dest.get(0).get(1).getValues()[0], "2");
+
+        assertEquals(dest.get(1).size(), 2);
+        assertEquals(dest.get(1).get(0).getName(), "from");
+        assertEquals(dest.get(1).get(0).getValues()[0], "3");
+        assertEquals(dest.get(1).get(1).getName(), "to");
+        assertEquals(dest.get(1).get(1).getValues()[0], "4");
+
+        assertEquals(dest.get(2).size(), 2);
+        assertEquals(dest.get(2).get(0).getName(), "from");
+        assertEquals(dest.get(2).get(0).getValues()[0], "5");
+        assertEquals(dest.get(2).get(1).getName(), "to");
+        assertEquals(dest.get(2).get(1).getValues()[0], "6");
+    }
+
+    @Test
+    public void testExpandFromToWithDefault() throws Exception {
+        Optional<Integer> nullValue = Optional.absent();
+
+        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), nullValue, 2, nullValue)
+                .expand();
+        assertEquals(dest.size(), 2);
+
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "from");
+        assertEquals(dest.get(0).get(0).getValues()[0], "0");
+        assertEquals(dest.get(0).get(1).getName(), "to");
+        assertEquals(dest.get(0).get(1).getValues()[0], "0");
+
+        assertEquals(dest.get(1).size(), 2);
+        assertEquals(dest.get(1).get(0).getName(), "from");
+        assertEquals(dest.get(1).get(0).getValues()[0], "1");
+        assertEquals(dest.get(1).get(1).getName(), "to");
+        assertEquals(dest.get(1).get(1).getValues()[0], "1");
+    }
+
+    @Test
+    public void testExpandPagenate() throws Exception {
+        Optional<String> nullValue = Optional.absent();
+        List<List<QueryConfig.Query>> dest = new PagerConfig("page", nullValue, Optional.of(1), 3,
+                Optional.of(1)).expand();
+        assertEquals(dest.size(), 3);
+
+        assertEquals(dest.get(0).size(), 1);
+        assertEquals(dest.get(0).get(0).getName(), "page");
+        assertEquals(dest.get(0).get(0).getValues()[0], "1");
+
+        assertEquals(dest.get(1).size(), 1);
+        assertEquals(dest.get(1).get(0).getName(), "page");
+        assertEquals(dest.get(1).get(0).getValues()[0], "2");
+
+        assertEquals(dest.get(2).size(), 1);
+        assertEquals(dest.get(2).get(0).getName(), "page");
+        assertEquals(dest.get(2).get(0).getValues()[0], "3");
+    }
+
+}

data/src/test/java/org/embulk/input/http/TestParamsConfig.java

@@ -0,0 +1,105 @@
+package org.embulk.input.http;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestParamsConfig {
+
+    @Test
+    public void testUnexpandQueriesSinglePair() throws Exception {
+        Optional<List<String>> nullValues = Optional.absent();
+        QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, false);
+        QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, false);
+        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
+        assertEquals(dest.size(), 1);
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "test1");
+        assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
+        assertEquals(dest.get(0).get(1).getName(), "test2");
+        assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
+    }
+
+    @Test
+    public void testUnexpandQueriesExpandPair() throws Exception {
+        Optional<String> nullValue = Optional.absent();
+        List<String> values1 = Lists.newArrayList("a", "b");
+        List<String> values2 = Lists.newArrayList("c", "d");
+
+        QueryConfig q1 = new QueryConfig("test1", nullValue, Optional.of(values1), false);
+        QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), false);
+
+        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
+        assertEquals(dest.size(), 1);
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "test1");
+        assertEquals(dest.get(0).get(0).getValues()[0], "a");
+        assertEquals(dest.get(0).get(0).getValues()[1], "b");
+        assertEquals(dest.get(0).get(1).getName(), "test2");
+        assertEquals(dest.get(0).get(1).getValues()[0], "c");
+        assertEquals(dest.get(0).get(1).getValues()[1], "d");
+    }
+
+    @Test
+    public void testExpandQueriesSinglePair() throws Exception {
+        Optional<List<String>> nullValues = Optional.absent();
+        QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, true);
+        QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, true);
+        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
+        assertEquals(dest.size(), 1);
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "test1");
+        assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
+        assertEquals(dest.get(0).get(1).getName(), "test2");
+        assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
+    }
+
+    @Test
+    public void testExpandQueriesExpandPair() throws Exception {
+        Optional<String> nullValue = Optional.absent();
+        List<String> values1 = Lists.newArrayList("a", "b");
+        List<String> values2 = Lists.newArrayList("c", "d");
+
+        QueryConfig q1 = new QueryConfig("test1", nullValue, Optional.of(values1), true);
+        QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), true);
+
+        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
+        assertEquals(dest.size(), 4);
+
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "test1");
+        assertEquals(dest.get(0).get(0).getValues()[0], "a");
+        assertEquals(dest.get(0).get(1).getName(), "test2");
+        assertEquals(dest.get(0).get(1).getValues()[0], "c");
+
+        assertEquals(dest.get(1).size(), 2);
+        assertEquals(dest.get(1).get(0).getName(), "test1");
+        assertEquals(dest.get(1).get(0).getValues()[0], "b");
+        assertEquals(dest.get(1).get(1).getName(), "test2");
+        assertEquals(dest.get(1).get(1).getValues()[0], "c");
+
+        assertEquals(dest.get(2).size(), 2);
+        assertEquals(dest.get(2).get(0).getName(), "test1");
+        assertEquals(dest.get(2).get(0).getValues()[0], "a");
+        assertEquals(dest.get(2).get(1).getName(), "test2");
+        assertEquals(dest.get(2).get(1).getValues()[0], "d");
+
+        assertEquals(dest.get(3).size(), 2);
+        assertEquals(dest.get(3).get(0).getName(), "test1");
+        assertEquals(dest.get(3).get(0).getValues()[0], "b");
+        assertEquals(dest.get(3).get(1).getName(), "test2");
+        assertEquals(dest.get(3).get(1).getValues()[0], "d");
+    }
+}