embulk-input-http 0.0.20 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +3 -3
- data/src/main/java/org/embulk/input/http/BasicAuthOption.java +25 -31
- data/src/main/java/org/embulk/input/http/HttpFileInputPlugin.java +274 -296
- data/src/main/java/org/embulk/input/http/PagerOption.java +73 -76
- data/src/main/java/org/embulk/input/http/ParamsOption.java +57 -66
- data/src/main/java/org/embulk/input/http/QueryOption.java +138 -161
- data/src/main/java/org/embulk/input/http/RetryableHandler.java +57 -63
- data/src/test/java/org/embulk/input/http/TestHttpInputPlugin.java +1 -3
- data/src/test/java/org/embulk/input/http/TestPagerOption.java +55 -59
- data/src/test/java/org/embulk/input/http/TestParamsOption.java +82 -87
- data/src/test/java/org/embulk/input/http/TestQueryOption.java +116 -121
- metadata +3 -3
@@ -11,82 +11,76 @@ import org.embulk.spi.util.RetryExecutor;
|
|
11
11
|
import org.slf4j.Logger;
|
12
12
|
|
13
13
|
import javax.net.ssl.SSLException;
|
14
|
-
|
15
14
|
import java.io.IOException;
|
16
15
|
import java.io.InterruptedIOException;
|
17
16
|
import java.net.UnknownHostException;
|
18
17
|
import java.util.List;
|
19
18
|
|
20
|
-
public class RetryableHandler implements RetryExecutor.Retryable
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
19
|
+
public class RetryableHandler implements RetryExecutor.Retryable {
|
20
|
+
private static final List<Class<? extends IOException>> NOT_RETRIABLE_CLASSES =
|
21
|
+
ImmutableList.of(
|
22
|
+
UnknownHostException.class, InterruptedIOException.class, SSLException.class);
|
23
|
+
protected final Logger logger = Exec.getLogger(getClass());
|
24
|
+
private final HttpClient client;
|
25
|
+
private final HttpRequestBase request;
|
26
|
+
private HttpResponse response;
|
26
27
|
|
27
|
-
|
28
|
-
|
29
|
-
|
28
|
+
public RetryableHandler(HttpClient client, HttpRequestBase request) {
|
29
|
+
this.client = client;
|
30
|
+
this.request = request;
|
31
|
+
}
|
30
32
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
this.request = request;
|
35
|
-
}
|
33
|
+
public HttpResponse getResponse() {
|
34
|
+
return response;
|
35
|
+
}
|
36
36
|
|
37
|
-
|
38
|
-
|
39
|
-
|
37
|
+
@Override
|
38
|
+
public Object call() throws Exception {
|
39
|
+
if (response != null) {
|
40
|
+
throw new IllegalStateException("response is already set");
|
40
41
|
}
|
42
|
+
HttpResponse response = client.execute(request);
|
43
|
+
statusIsOkOrThrow(response);
|
44
|
+
this.response = response;
|
45
|
+
return null;
|
46
|
+
}
|
41
47
|
|
42
|
-
|
43
|
-
|
44
|
-
{
|
45
|
-
|
46
|
-
|
47
|
-
}
|
48
|
-
HttpResponse response = client.execute(request);
|
49
|
-
statusIsOkOrThrow(response);
|
50
|
-
this.response = response;
|
51
|
-
return null;
|
48
|
+
@Override
|
49
|
+
public boolean isRetryableException(Exception exception) {
|
50
|
+
if (NOT_RETRIABLE_CLASSES.contains(exception.getClass())) {
|
51
|
+
logger.error(String.format("'%s' is not retriable", exception.getClass()));
|
52
|
+
return false;
|
52
53
|
}
|
54
|
+
return true;
|
55
|
+
}
|
53
56
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
57
|
+
@Override
|
58
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
59
|
+
throws RetryExecutor.RetryGiveupException {
|
60
|
+
logger.warn(
|
61
|
+
"retrying {}/{} after {} seconds. Message: {}",
|
62
|
+
retryCount,
|
63
|
+
retryLimit,
|
64
|
+
retryWait / 1000,
|
65
|
+
exception.getMessage());
|
66
|
+
}
|
63
67
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
{
|
68
|
-
|
69
|
-
retryCount, retryLimit, retryWait / 1000,
|
70
|
-
exception.getMessage());
|
71
|
-
}
|
72
|
-
|
73
|
-
@Override
|
74
|
-
public void onGiveup(Exception firstException, Exception lastException)
|
75
|
-
throws RetryExecutor.RetryGiveupException
|
76
|
-
{
|
77
|
-
logger.error("giveup {}", lastException.getMessage());
|
78
|
-
}
|
68
|
+
@Override
|
69
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
70
|
+
throws RetryExecutor.RetryGiveupException {
|
71
|
+
logger.error("giveup {}", lastException.getMessage());
|
72
|
+
}
|
79
73
|
|
80
|
-
|
81
|
-
|
82
|
-
{
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
}
|
74
|
+
protected void statusIsOkOrThrow(HttpResponse response) throws HttpException, IOException {
|
75
|
+
int code = response.getStatusLine().getStatusCode();
|
76
|
+
switch (response.getStatusLine().getStatusCode()) {
|
77
|
+
case 200:
|
78
|
+
return;
|
79
|
+
default:
|
80
|
+
throw new HttpException(
|
81
|
+
String.format(
|
82
|
+
"Request is not successful, code=%d, body=%s",
|
83
|
+
code, EntityUtils.toString(response.getEntity())));
|
91
84
|
}
|
85
|
+
}
|
92
86
|
}
|
@@ -7,74 +7,70 @@ import java.util.List;
|
|
7
7
|
|
8
8
|
import static org.junit.Assert.assertEquals;
|
9
9
|
|
10
|
-
public class TestPagerOption
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
Optional.of(2)).expand();
|
17
|
-
assertEquals(dest.size(), 3);
|
10
|
+
public class TestPagerOption {
|
11
|
+
@Test
|
12
|
+
public void testExpandFromTo() throws Exception {
|
13
|
+
List<List<QueryOption.Query>> dest =
|
14
|
+
new PagerOption("from", Optional.of("to"), Optional.of(1), 3, Optional.of(2)).expand();
|
15
|
+
assertEquals(dest.size(), 3);
|
18
16
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
17
|
+
assertEquals(dest.get(0).size(), 2);
|
18
|
+
assertEquals(dest.get(0).get(0).getName(), "from");
|
19
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "1");
|
20
|
+
assertEquals(dest.get(0).get(1).getName(), "to");
|
21
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "2");
|
24
22
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
23
|
+
assertEquals(dest.get(1).size(), 2);
|
24
|
+
assertEquals(dest.get(1).get(0).getName(), "from");
|
25
|
+
assertEquals(dest.get(1).get(0).getValues()[0], "3");
|
26
|
+
assertEquals(dest.get(1).get(1).getName(), "to");
|
27
|
+
assertEquals(dest.get(1).get(1).getValues()[0], "4");
|
30
28
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
29
|
+
assertEquals(dest.get(2).size(), 2);
|
30
|
+
assertEquals(dest.get(2).get(0).getName(), "from");
|
31
|
+
assertEquals(dest.get(2).get(0).getValues()[0], "5");
|
32
|
+
assertEquals(dest.get(2).get(1).getName(), "to");
|
33
|
+
assertEquals(dest.get(2).get(1).getValues()[0], "6");
|
34
|
+
}
|
37
35
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
Optional<Integer> nullValue = Optional.absent();
|
36
|
+
@Test
|
37
|
+
public void testExpandFromToWithDefault() throws Exception {
|
38
|
+
Optional<Integer> nullValue = Optional.absent();
|
42
39
|
|
43
|
-
|
44
|
-
|
45
|
-
|
40
|
+
List<List<QueryOption.Query>> dest =
|
41
|
+
new PagerOption("from", Optional.of("to"), nullValue, 2, nullValue).expand();
|
42
|
+
assertEquals(dest.size(), 2);
|
46
43
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
44
|
+
assertEquals(dest.get(0).size(), 2);
|
45
|
+
assertEquals(dest.get(0).get(0).getName(), "from");
|
46
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "0");
|
47
|
+
assertEquals(dest.get(0).get(1).getName(), "to");
|
48
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "0");
|
52
49
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
50
|
+
assertEquals(dest.get(1).size(), 2);
|
51
|
+
assertEquals(dest.get(1).get(0).getName(), "from");
|
52
|
+
assertEquals(dest.get(1).get(0).getValues()[0], "1");
|
53
|
+
assertEquals(dest.get(1).get(1).getName(), "to");
|
54
|
+
assertEquals(dest.get(1).get(1).getValues()[0], "1");
|
55
|
+
}
|
59
56
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
assertEquals(dest.size(), 3);
|
57
|
+
@Test
|
58
|
+
public void testExpandPagenate() throws Exception {
|
59
|
+
Optional<String> nullValue = Optional.absent();
|
60
|
+
List<List<QueryOption.Query>> dest =
|
61
|
+
new PagerOption("page", nullValue, Optional.of(1), 3, Optional.of(1)).expand();
|
62
|
+
assertEquals(dest.size(), 3);
|
67
63
|
|
68
|
-
|
69
|
-
|
70
|
-
|
64
|
+
assertEquals(dest.get(0).size(), 1);
|
65
|
+
assertEquals(dest.get(0).get(0).getName(), "page");
|
66
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "1");
|
71
67
|
|
72
|
-
|
73
|
-
|
74
|
-
|
68
|
+
assertEquals(dest.get(1).size(), 1);
|
69
|
+
assertEquals(dest.get(1).get(0).getName(), "page");
|
70
|
+
assertEquals(dest.get(1).get(0).getValues()[0], "2");
|
75
71
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
72
|
+
assertEquals(dest.get(2).size(), 1);
|
73
|
+
assertEquals(dest.get(2).get(0).getName(), "page");
|
74
|
+
assertEquals(dest.get(2).get(0).getValues()[0], "3");
|
75
|
+
}
|
80
76
|
}
|
@@ -8,102 +8,97 @@ import java.util.List;
|
|
8
8
|
|
9
9
|
import static org.junit.Assert.assertEquals;
|
10
10
|
|
11
|
-
public class TestParamsOption
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
|
28
|
-
}
|
11
|
+
public class TestParamsOption {
|
12
|
+
@Test
|
13
|
+
public void testUnexpandQueriesSinglePair() throws Exception {
|
14
|
+
Optional<List<String>> nullValues = Optional.absent();
|
15
|
+
QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, false);
|
16
|
+
QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, false);
|
17
|
+
ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
|
18
|
+
Optional<PagerOption> pagerOption = Optional.absent();
|
19
|
+
List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
|
20
|
+
assertEquals(dest.size(), 1);
|
21
|
+
assertEquals(dest.get(0).size(), 2);
|
22
|
+
assertEquals(dest.get(0).get(0).getName(), "test1");
|
23
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
|
24
|
+
assertEquals(dest.get(0).get(1).getName(), "test2");
|
25
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
|
26
|
+
}
|
29
27
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
List<String> values2 = Lists.newArrayList("c", "d");
|
28
|
+
@Test
|
29
|
+
public void testUnexpandQueriesExpandPair() throws Exception {
|
30
|
+
Optional<String> nullValue = Optional.absent();
|
31
|
+
List<String> values1 = Lists.newArrayList("a", "b");
|
32
|
+
List<String> values2 = Lists.newArrayList("c", "d");
|
36
33
|
|
37
|
-
|
38
|
-
|
34
|
+
QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), false);
|
35
|
+
QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), false);
|
39
36
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
37
|
+
ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
|
38
|
+
Optional<PagerOption> pagerOption = Optional.absent();
|
39
|
+
List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
|
40
|
+
assertEquals(dest.size(), 1);
|
41
|
+
assertEquals(dest.get(0).size(), 2);
|
42
|
+
assertEquals(dest.get(0).get(0).getName(), "test1");
|
43
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "a");
|
44
|
+
assertEquals(dest.get(0).get(0).getValues()[1], "b");
|
45
|
+
assertEquals(dest.get(0).get(1).getName(), "test2");
|
46
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "c");
|
47
|
+
assertEquals(dest.get(0).get(1).getValues()[1], "d");
|
48
|
+
}
|
52
49
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
}
|
50
|
+
@Test
|
51
|
+
public void testExpandQueriesSinglePair() throws Exception {
|
52
|
+
Optional<List<String>> nullValues = Optional.absent();
|
53
|
+
QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, true);
|
54
|
+
QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, true);
|
55
|
+
ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
|
56
|
+
Optional<PagerOption> pagerOption = Optional.absent();
|
57
|
+
List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
|
58
|
+
assertEquals(dest.size(), 1);
|
59
|
+
assertEquals(dest.get(0).size(), 2);
|
60
|
+
assertEquals(dest.get(0).get(0).getName(), "test1");
|
61
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
|
62
|
+
assertEquals(dest.get(0).get(1).getName(), "test2");
|
63
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
|
64
|
+
}
|
69
65
|
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
List<String> values2 = Lists.newArrayList("c", "d");
|
66
|
+
@Test
|
67
|
+
public void testExpandQueriesExpandPair() throws Exception {
|
68
|
+
Optional<String> nullValue = Optional.absent();
|
69
|
+
List<String> values1 = Lists.newArrayList("a", "b");
|
70
|
+
List<String> values2 = Lists.newArrayList("c", "d");
|
76
71
|
|
77
|
-
|
78
|
-
|
72
|
+
QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), true);
|
73
|
+
QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), true);
|
79
74
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
75
|
+
ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
|
76
|
+
Optional<PagerOption> pagerOption = Optional.absent();
|
77
|
+
List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
|
78
|
+
assertEquals(dest.size(), 4);
|
84
79
|
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
80
|
+
assertEquals(dest.get(0).size(), 2);
|
81
|
+
assertEquals(dest.get(0).get(0).getName(), "test1");
|
82
|
+
assertEquals(dest.get(0).get(0).getValues()[0], "a");
|
83
|
+
assertEquals(dest.get(0).get(1).getName(), "test2");
|
84
|
+
assertEquals(dest.get(0).get(1).getValues()[0], "c");
|
90
85
|
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
86
|
+
assertEquals(dest.get(1).size(), 2);
|
87
|
+
assertEquals(dest.get(1).get(0).getName(), "test1");
|
88
|
+
assertEquals(dest.get(1).get(0).getValues()[0], "b");
|
89
|
+
assertEquals(dest.get(1).get(1).getName(), "test2");
|
90
|
+
assertEquals(dest.get(1).get(1).getValues()[0], "c");
|
96
91
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
92
|
+
assertEquals(dest.get(2).size(), 2);
|
93
|
+
assertEquals(dest.get(2).get(0).getName(), "test1");
|
94
|
+
assertEquals(dest.get(2).get(0).getValues()[0], "a");
|
95
|
+
assertEquals(dest.get(2).get(1).getName(), "test2");
|
96
|
+
assertEquals(dest.get(2).get(1).getValues()[0], "d");
|
102
97
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
98
|
+
assertEquals(dest.get(3).size(), 2);
|
99
|
+
assertEquals(dest.get(3).get(0).getName(), "test1");
|
100
|
+
assertEquals(dest.get(3).get(0).getValues()[0], "b");
|
101
|
+
assertEquals(dest.get(3).get(1).getName(), "test2");
|
102
|
+
assertEquals(dest.get(3).get(1).getValues()[0], "d");
|
103
|
+
}
|
109
104
|
}
|