embulk-input-http 0.0.13 → 0.0.14

data/src/main/java/org/embulk/input/PagerConfig.java

@@ -1,86 +0,0 @@
-package org.embulk.input;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class PagerConfig {
-
-    private final String fromParam;
-    private final Optional<String> toParam;
-    private final int start;
-    private final int pages;
-    private final int step;
-
-    @JsonCreator
-    public PagerConfig(@JsonProperty("from_param") String fromParam,
-            @JsonProperty("to_param") Optional<String> toParam,
-            @JsonProperty("start") Optional<Integer> start,
-            @JsonProperty("pages") int pages,
-            @JsonProperty("step") Optional<Integer> step) {
-        this.fromParam = fromParam;
-        this.toParam = toParam;
-        this.start = start.or(0);
-        this.pages = pages;
-        this.step = step.or(1);
-    }
-
-    public List<List<QueryConfig.Query>> expand() {
-        List<List<QueryConfig.Query>> queries = new ArrayList<>();
-        int p = 1;
-        int index = start;
-        while (p <= pages) {
-            List<QueryConfig.Query> one = new ArrayList<>();
-            one.add(new QueryConfig.Query(fromParam, Integer.toString(index)));
-            if (toParam.isPresent()) {
-                int t = index + step - 1;
-                one.add(new QueryConfig.Query(toParam.get(), Integer.toString(t)));
-                index = t + 1;
-            } else {
-                index += step;
-            }
-            queries.add(one);
-            p++;
-        }
-        return queries;
-    }
-
-    @JsonProperty("from_param")
-    public String getFromParam() {
-        return fromParam;
-    }
-
-    @JsonProperty("to_param")
-    public Optional<String> getToParam() {
-        return toParam;
-    }
-
-    @JsonProperty("start")
-    public int getStart() {
-        return start;
-    }
-
-    @JsonProperty("pages")
-    public int getPages() {
-        return pages;
-    }
-
-    @JsonProperty("step")
-    public int getStep() {
-        return step;
-    }
-
-    @Override
-    public String toString() {
-        return "PagerConfig{" +
-                "fromParam='" + fromParam + '\'' +
-                ", toParam=" + toParam +
-                ", start=" + start +
-                ", pages=" + pages +
-                ", step=" + step +
-                '}';
-    }
-}

data/src/main/java/org/embulk/input/ParamsConfig.java

@@ -1,82 +0,0 @@
-package org.embulk.input;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ParamsConfig {
-
-    private final List<QueryConfig> queries;
-
-    @JsonCreator
-    public ParamsConfig(List<QueryConfig> queries) {
-        this.queries = queries;
-    }
-
-    @JsonValue
-    public List<QueryConfig> getQueries() {
-        return queries;
-    }
-
-    public List<List<QueryConfig.Query>> generateQueries(Optional<PagerConfig> pagerConfig) {
-        List<List<QueryConfig.Query>> base = new ArrayList<>(queries.size());
-        for (QueryConfig p : queries) {
-            base.add(p.expand());
-        }
-
-        int productSize = 1;
-        int baseSize = base.size();
-        for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++);
-
-        List<List<QueryConfig.Query>> expands = new ArrayList<>(productSize);
-        for (int i = 0; i < productSize; i++) {
-            int j = 1;
-            List<QueryConfig.Query> one = new ArrayList<>();
-            for (List<QueryConfig.Query> list : base) {
-                QueryConfig.Query pc = list.get((i / j) % list.size());
-                one.add(pc);
-                j *= list.size();
-            }
-            if (pagerConfig.isPresent()) {
-                for (List<QueryConfig.Query> q : pagerConfig.get().expand()) {
-                    expands.add(copyAndConcat(one, q));
-                }
-            } else {
-                expands.add(one);
-            }
-        }
-
-        return expands;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (!(obj instanceof ParamsConfig)) {
-            return false;
-        }
-        ParamsConfig other = (ParamsConfig) obj;
-        return Objects.equal(queries, other.queries);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hashCode(queries);
-    }
-
-    private List<QueryConfig.Query> copyAndConcat(List<QueryConfig.Query>... srcs) {
-        List<QueryConfig.Query> dest = new ArrayList<>();
-        for (List<QueryConfig.Query> src : srcs) {
-            for (QueryConfig.Query q : src) {
-                dest.add(q.copy());
-            }
-        }
-        return dest;
-    }
-}

data/src/main/java/org/embulk/input/QueryConfig.java

@@ -1,171 +0,0 @@
-package org.embulk.input;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Objects;
-import com.google.common.base.Optional;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class QueryConfig {
-
-    private final String name;
-    private final Optional<String> value;
-    private final Optional<List<String>> values;
-    private final boolean expand;
-
-    @JsonCreator
-    public QueryConfig(
-            @JsonProperty("name") String name,
-            @JsonProperty("value") Optional<String> value,
-            @JsonProperty("values") Optional<List<String>> values,
-            @JsonProperty("expand") boolean expand) {
-        this.name = name;
-        this.value = value;
-        this.values = values;
-        this.expand = expand;
-    }
-
-    public List<Query> expand() {
-        List<Query> dest;
-        if (value.isPresent()) {
-            if (expand) {
-                List<String> expanded = BraceExpansion.expand(value.get());
-                dest = new ArrayList<>(expanded.size());
-                for (String s : expanded) {
-                    dest.add(new Query(name, s));
-                }
-            } else {
-                dest = new ArrayList<>(1);
-                dest.add(new Query(name, value.get()));
-            }
-        } else if (values.isPresent()) {
-            if (expand) {
-                dest = new ArrayList<>(values.get().size());
-                for (String s : values.get()) {
-                    dest.add(new Query(name, s));
-                }
-            } else {
-                dest = new ArrayList<>(1);
-                final String[] valueArr = values.get().toArray(new String[values.get().size()]);
-                dest.add(new Query(name, valueArr));
-            }
-        } else {
-            throw new IllegalArgumentException("value or values must be specified to 'params'");
-        }
-        return dest;
-    }
-
-    @JsonProperty("name")
-    public String getName() {
-        return name;
-    }
-
-    @JsonProperty("value")
-    public Optional<String> getValue() {
-        return value;
-    }
-
-    @JsonProperty("expand")
-    public boolean isExpand() {
-        return expand;
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (!(obj instanceof QueryConfig)) {
-            return false;
-        }
-        QueryConfig other = (QueryConfig) obj;
-        return Objects.equal(this.name, other.name) &&
-                Objects.equal(value, other.value) &&
-                Objects.equal(expand, other.expand);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hashCode(name, value, expand);
-    }
-
-    @Override
-    public String toString() {
-        return String.format("ParameterConfig[%s, %s, %s]",
-                getName(), getValue(), isExpand());
-    }
-
-    public static class Query {
-        private final String name;
-        private final String[] values;
-
-        public Query(
-                @JsonProperty("name") String name,
-                @JsonProperty("values") String... values) {
-            this.name = name;
-            this.values = values;
-        }
-
-        public String getName() {
-            return name;
-        }
-
-        public String[] getValues() {
-            return values;
-        }
-
-        public Query copy() {
-            return new Query(this.name, Arrays.copyOf(this.values, this.values.length));
-        }
-    }
-
-    private static class BraceExpansion {
-
-        public static List<String> expand(String s) {
-            return expandRecursive("", s, "", new ArrayList<String>());
-        }
-
-        private static List<String> expandRecursive(String prefix, String s,
-                                                    String suffix, List<String> dest) {
-            // I used the code below as reference.
-            //  http://rosettacode.org/wiki/Brace_expansion#Java
-            int i1 = -1, i2 = 0;
-            String noEscape = s.replaceAll("([\\\\]{2}|[\\\\][,}{])", "  ");
-            StringBuilder sb = null;
-
-            outer:
-            while ((i1 = noEscape.indexOf('{', i1 + 1)) != -1) {
-                i2 = i1 + 1;
-                sb = new StringBuilder(s);
-                for (int depth = 1; i2 < s.length() && depth > 0; i2++) {
-                    char c = noEscape.charAt(i2);
-                    depth = (c == '{') ? ++depth : depth;
-                    depth = (c == '}') ? --depth : depth;
-                    if (c == ',' && depth == 1) {
-                        sb.setCharAt(i2, '\u0000');
-                    } else if (c == '}' && depth == 0 && sb.indexOf("\u0000") != -1) {
-                        break outer;
-                    }
-                }
-            }
-
-            if (i1 == -1) {
-                if (suffix.length() > 0) {
-                    expandRecursive(prefix + s, suffix, "", dest);
-                } else {
-                    final String out = String.format("%s%s%s", prefix, s, suffix).
-                            replaceAll("[\\\\]{2}", "\\").replaceAll("[\\\\]([,}{])", "$1");
-                    dest.add(out);
-                }
-            } else {
-                for (String m : sb.substring(i1 + 1, i2).split("\u0000", -1)) {
-                    expandRecursive(prefix + s.substring(0, i1), m, s.substring(i2 + 1) + suffix, dest);
-                }
-            }
-            return dest;
-        }
-    }
-}

data/src/main/java/org/embulk/input/RetryableHandler.java

@@ -1,90 +0,0 @@
-package org.embulk.input;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.http.HttpException;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.util.EntityUtils;
-import org.embulk.spi.Exec;
-import org.embulk.spi.util.RetryExecutor;
-import org.slf4j.Logger;
-
-import javax.net.ssl.SSLException;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.UnknownHostException;
-import java.util.List;
-
-public class RetryableHandler implements RetryExecutor.Retryable {
-
-    protected final Logger logger = Exec.getLogger(getClass());
-
-    private static List<Class<? extends IOException>> NOT_RETRIABLE_CLAASSES;
-
-    private final HttpClient client;
-    private final HttpRequestBase request;
-    private HttpResponse response;
-
-    static {
-        ImmutableList.Builder<Class<? extends IOException>> classes = ImmutableList.builder();
-        classes.add(UnknownHostException.class).
-                add(InterruptedIOException.class).
-                add(SSLException.class);
-        NOT_RETRIABLE_CLAASSES = classes.build();
-    }
-
-    public RetryableHandler(HttpClient client, HttpRequestBase request) {
-        this.client = client;
-        this.request = request;
-    }
-
-    public HttpResponse getResponse() {
-        return response;
-    }
-
-    @Override
-    public Object call() throws Exception {
-        if (response != null) throw new IllegalStateException("response is already set");
-        HttpResponse response = client.execute(request);
-        statusIsOkOrThrow(response);
-        this.response = response;
-        return null;
-    }
-
-    @Override
-    public boolean isRetryableException(Exception exception) {
-        if (NOT_RETRIABLE_CLAASSES.contains(exception.getClass())) {
-            logger.error(String.format("'%s' is not retriable", exception.getClass()));
-            return false;
-        }
-        return true;
-    }
-
-    @Override
-    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-            throws RetryExecutor.RetryGiveupException {
-        logger.warn("retrying {}/{} after {} seconds. Message: {}",
-                retryCount, retryLimit, retryWait / 1000,
-                exception.getMessage());
-    }
-
-    @Override
-    public void onGiveup(Exception firstException, Exception lastException)
-            throws RetryExecutor.RetryGiveupException {
-        logger.error("giveup {}", lastException.getMessage());
-    }
-
-    protected void statusIsOkOrThrow(HttpResponse response)
-            throws HttpException, IOException {
-        int code = response.getStatusLine().getStatusCode();
-        switch (response.getStatusLine().getStatusCode()) {
-            case 200:
-                return;
-            default:
-                throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
-                        code, EntityUtils.toString(response.getEntity())));
-        }
-    }
-
-}

data/src/test/java/org/embulk/input/TestHttpInputPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.input;
-
-public class TestHttpInputPlugin
-{
-}

data/src/test/java/org/embulk/input/TestPagerConfig.java

@@ -1,78 +0,0 @@
-package org.embulk.input;
-
-import com.google.common.base.Optional;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestPagerConfig {
-
-    @Test
-    public void testExpandFromTo() throws Exception {
-        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), Optional.of(1), 3,
-                Optional.of(2)).expand();
-        assertEquals(dest.size(), 3);
-
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "from");
-        assertEquals(dest.get(0).get(0).getValues()[0], "1");
-        assertEquals(dest.get(0).get(1).getName(), "to");
-        assertEquals(dest.get(0).get(1).getValues()[0], "2");
-
-        assertEquals(dest.get(1).size(), 2);
-        assertEquals(dest.get(1).get(0).getName(), "from");
-        assertEquals(dest.get(1).get(0).getValues()[0], "3");
-        assertEquals(dest.get(1).get(1).getName(), "to");
-        assertEquals(dest.get(1).get(1).getValues()[0], "4");
-
-        assertEquals(dest.get(2).size(), 2);
-        assertEquals(dest.get(2).get(0).getName(), "from");
-        assertEquals(dest.get(2).get(0).getValues()[0], "5");
-        assertEquals(dest.get(2).get(1).getName(), "to");
-        assertEquals(dest.get(2).get(1).getValues()[0], "6");
-    }
-
-    @Test
-    public void testExpandFromToWithDefault() throws Exception {
-        Optional<Integer> nullValue = Optional.absent();
-
-        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), nullValue, 2, nullValue)
-                .expand();
-        assertEquals(dest.size(), 2);
-
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "from");
-        assertEquals(dest.get(0).get(0).getValues()[0], "0");
-        assertEquals(dest.get(0).get(1).getName(), "to");
-        assertEquals(dest.get(0).get(1).getValues()[0], "0");
-
-        assertEquals(dest.get(1).size(), 2);
-        assertEquals(dest.get(1).get(0).getName(), "from");
-        assertEquals(dest.get(1).get(0).getValues()[0], "1");
-        assertEquals(dest.get(1).get(1).getName(), "to");
-        assertEquals(dest.get(1).get(1).getValues()[0], "1");
-    }
-
-    @Test
-    public void testExpandPagenate() throws Exception {
-        Optional<String> nullValue = Optional.absent();
-        List<List<QueryConfig.Query>> dest = new PagerConfig("page", nullValue, Optional.of(1), 3,
-                Optional.of(1)).expand();
-        assertEquals(dest.size(), 3);
-
-        assertEquals(dest.get(0).size(), 1);
-        assertEquals(dest.get(0).get(0).getName(), "page");
-        assertEquals(dest.get(0).get(0).getValues()[0], "1");
-
-        assertEquals(dest.get(1).size(), 1);
-        assertEquals(dest.get(1).get(0).getName(), "page");
-        assertEquals(dest.get(1).get(0).getValues()[0], "2");
-
-        assertEquals(dest.get(2).size(), 1);
-        assertEquals(dest.get(2).get(0).getName(), "page");
-        assertEquals(dest.get(2).get(0).getValues()[0], "3");
-    }
-
-}

data/src/test/java/org/embulk/input/TestParamsConfig.java

@@ -1,105 +0,0 @@
-package org.embulk.input;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.Lists;
-import org.junit.Test;
-
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-
-public class TestParamsConfig {
-
-    @Test
-    public void testUnexpandQueriesSinglePair() throws Exception {
-        Optional<List<String>> nullValues = Optional.absent();
-        QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, false);
-        QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, false);
-        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        Optional<PagerConfig> pagerConfig = Optional.absent();
-        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
-        assertEquals(dest.size(), 1);
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "test1");
-        assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
-        assertEquals(dest.get(0).get(1).getName(), "test2");
-        assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
-    }
-
-    @Test
-    public void testUnexpandQueriesExpandPair() throws Exception {
-        Optional<String> nullValue = Optional.absent();
-        List<String> values1 = Lists.newArrayList("a", "b");
-        List<String> values2 = Lists.newArrayList("c", "d");
-
-        QueryConfig q1 = new QueryConfig("test1", nullValue, Optional.of(values1), false);
-        QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), false);
-
-        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        Optional<PagerConfig> pagerConfig = Optional.absent();
-        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
-        assertEquals(dest.size(), 1);
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "test1");
-        assertEquals(dest.get(0).get(0).getValues()[0], "a");
-        assertEquals(dest.get(0).get(0).getValues()[1], "b");
-        assertEquals(dest.get(0).get(1).getName(), "test2");
-        assertEquals(dest.get(0).get(1).getValues()[0], "c");
-        assertEquals(dest.get(0).get(1).getValues()[1], "d");
-    }
-
-    @Test
-    public void testExpandQueriesSinglePair() throws Exception {
-        Optional<List<String>> nullValues = Optional.absent();
-        QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, true);
-        QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, true);
-        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        Optional<PagerConfig> pagerConfig = Optional.absent();
-        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
-        assertEquals(dest.size(), 1);
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "test1");
-        assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
-        assertEquals(dest.get(0).get(1).getName(), "test2");
-        assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
-    }
-
-    @Test
-    public void testExpandQueriesExpandPair() throws Exception {
-        Optional<String> nullValue = Optional.absent();
-        List<String> values1 = Lists.newArrayList("a", "b");
-        List<String> values2 = Lists.newArrayList("c", "d");
-
-        QueryConfig q1 = new QueryConfig("test1", nullValue, Optional.of(values1), true);
-        QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), true);
-
-        ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        Optional<PagerConfig> pagerConfig = Optional.absent();
-        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
-        assertEquals(dest.size(), 4);
-
-        assertEquals(dest.get(0).size(), 2);
-        assertEquals(dest.get(0).get(0).getName(), "test1");
-        assertEquals(dest.get(0).get(0).getValues()[0], "a");
-        assertEquals(dest.get(0).get(1).getName(), "test2");
-        assertEquals(dest.get(0).get(1).getValues()[0], "c");
-
-        assertEquals(dest.get(1).size(), 2);
-        assertEquals(dest.get(1).get(0).getName(), "test1");
-        assertEquals(dest.get(1).get(0).getValues()[0], "b");
-        assertEquals(dest.get(1).get(1).getName(), "test2");
-        assertEquals(dest.get(1).get(1).getValues()[0], "c");
-
-        assertEquals(dest.get(2).size(), 2);
-        assertEquals(dest.get(2).get(0).getName(), "test1");
-        assertEquals(dest.get(2).get(0).getValues()[0], "a");
-        assertEquals(dest.get(2).get(1).getName(), "test2");
-        assertEquals(dest.get(2).get(1).getValues()[0], "d");
-
-        assertEquals(dest.get(3).size(), 2);
-        assertEquals(dest.get(3).get(0).getName(), "test1");
-        assertEquals(dest.get(3).get(0).getValues()[0], "b");
-        assertEquals(dest.get(3).get(1).getName(), "test2");
-        assertEquals(dest.get(3).get(1).getValues()[0], "d");
-    }
-}