embulk-input-http 0.0.20 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,82 +11,76 @@ import org.embulk.spi.util.RetryExecutor;
11
11
  import org.slf4j.Logger;
12
12
 
13
13
  import javax.net.ssl.SSLException;
14
-
15
14
  import java.io.IOException;
16
15
  import java.io.InterruptedIOException;
17
16
  import java.net.UnknownHostException;
18
17
  import java.util.List;
19
18
 
20
- public class RetryableHandler implements RetryExecutor.Retryable
21
- {
22
- protected final Logger logger = Exec.getLogger(getClass());
23
-
24
- private static final List<Class<? extends IOException>> NOT_RETRIABLE_CLASSES = ImmutableList.of(UnknownHostException.class,
25
- InterruptedIOException.class, SSLException.class);
19
+ public class RetryableHandler implements RetryExecutor.Retryable {
20
+ private static final List<Class<? extends IOException>> NOT_RETRIABLE_CLASSES =
21
+ ImmutableList.of(
22
+ UnknownHostException.class, InterruptedIOException.class, SSLException.class);
23
+ protected final Logger logger = Exec.getLogger(getClass());
24
+ private final HttpClient client;
25
+ private final HttpRequestBase request;
26
+ private HttpResponse response;
26
27
 
27
- private final HttpClient client;
28
- private final HttpRequestBase request;
29
- private HttpResponse response;
28
+ public RetryableHandler(HttpClient client, HttpRequestBase request) {
29
+ this.client = client;
30
+ this.request = request;
31
+ }
30
32
 
31
- public RetryableHandler(HttpClient client, HttpRequestBase request)
32
- {
33
- this.client = client;
34
- this.request = request;
35
- }
33
+ public HttpResponse getResponse() {
34
+ return response;
35
+ }
36
36
 
37
- public HttpResponse getResponse()
38
- {
39
- return response;
37
+ @Override
38
+ public Object call() throws Exception {
39
+ if (response != null) {
40
+ throw new IllegalStateException("response is already set");
40
41
  }
42
+ HttpResponse response = client.execute(request);
43
+ statusIsOkOrThrow(response);
44
+ this.response = response;
45
+ return null;
46
+ }
41
47
 
42
- @Override
43
- public Object call() throws Exception
44
- {
45
- if (response != null) {
46
- throw new IllegalStateException("response is already set");
47
- }
48
- HttpResponse response = client.execute(request);
49
- statusIsOkOrThrow(response);
50
- this.response = response;
51
- return null;
48
+ @Override
49
+ public boolean isRetryableException(Exception exception) {
50
+ if (NOT_RETRIABLE_CLASSES.contains(exception.getClass())) {
51
+ logger.error(String.format("'%s' is not retriable", exception.getClass()));
52
+ return false;
52
53
  }
54
+ return true;
55
+ }
53
56
 
54
- @Override
55
- public boolean isRetryableException(Exception exception)
56
- {
57
- if (NOT_RETRIABLE_CLASSES.contains(exception.getClass())) {
58
- logger.error(String.format("'%s' is not retriable", exception.getClass()));
59
- return false;
60
- }
61
- return true;
62
- }
57
+ @Override
58
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
59
+ throws RetryExecutor.RetryGiveupException {
60
+ logger.warn(
61
+ "retrying {}/{} after {} seconds. Message: {}",
62
+ retryCount,
63
+ retryLimit,
64
+ retryWait / 1000,
65
+ exception.getMessage());
66
+ }
63
67
 
64
- @Override
65
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
66
- throws RetryExecutor.RetryGiveupException
67
- {
68
- logger.warn("retrying {}/{} after {} seconds. Message: {}",
69
- retryCount, retryLimit, retryWait / 1000,
70
- exception.getMessage());
71
- }
72
-
73
- @Override
74
- public void onGiveup(Exception firstException, Exception lastException)
75
- throws RetryExecutor.RetryGiveupException
76
- {
77
- logger.error("giveup {}", lastException.getMessage());
78
- }
68
+ @Override
69
+ public void onGiveup(Exception firstException, Exception lastException)
70
+ throws RetryExecutor.RetryGiveupException {
71
+ logger.error("giveup {}", lastException.getMessage());
72
+ }
79
73
 
80
- protected void statusIsOkOrThrow(HttpResponse response)
81
- throws HttpException, IOException
82
- {
83
- int code = response.getStatusLine().getStatusCode();
84
- switch (response.getStatusLine().getStatusCode()) {
85
- case 200:
86
- return;
87
- default:
88
- throw new HttpException(String.format("Request is not successful, code=%d, body=%s",
89
- code, EntityUtils.toString(response.getEntity())));
90
- }
74
+ protected void statusIsOkOrThrow(HttpResponse response) throws HttpException, IOException {
75
+ int code = response.getStatusLine().getStatusCode();
76
+ switch (response.getStatusLine().getStatusCode()) {
77
+ case 200:
78
+ return;
79
+ default:
80
+ throw new HttpException(
81
+ String.format(
82
+ "Request is not successful, code=%d, body=%s",
83
+ code, EntityUtils.toString(response.getEntity())));
91
84
  }
85
+ }
92
86
  }
@@ -1,5 +1,3 @@
1
1
  package org.embulk.input.http;
2
2
 
3
- public class TestHttpInputPlugin
4
- {
5
- }
3
+ public class TestHttpInputPlugin {}
@@ -7,74 +7,70 @@ import java.util.List;
7
7
 
8
8
  import static org.junit.Assert.assertEquals;
9
9
 
10
- public class TestPagerOption
11
- {
12
- @Test
13
- public void testExpandFromTo() throws Exception
14
- {
15
- List<List<QueryOption.Query>> dest = new PagerOption("from", Optional.of("to"), Optional.of(1), 3,
16
- Optional.of(2)).expand();
17
- assertEquals(dest.size(), 3);
10
+ public class TestPagerOption {
11
+ @Test
12
+ public void testExpandFromTo() throws Exception {
13
+ List<List<QueryOption.Query>> dest =
14
+ new PagerOption("from", Optional.of("to"), Optional.of(1), 3, Optional.of(2)).expand();
15
+ assertEquals(dest.size(), 3);
18
16
 
19
- assertEquals(dest.get(0).size(), 2);
20
- assertEquals(dest.get(0).get(0).getName(), "from");
21
- assertEquals(dest.get(0).get(0).getValues()[0], "1");
22
- assertEquals(dest.get(0).get(1).getName(), "to");
23
- assertEquals(dest.get(0).get(1).getValues()[0], "2");
17
+ assertEquals(dest.get(0).size(), 2);
18
+ assertEquals(dest.get(0).get(0).getName(), "from");
19
+ assertEquals(dest.get(0).get(0).getValues()[0], "1");
20
+ assertEquals(dest.get(0).get(1).getName(), "to");
21
+ assertEquals(dest.get(0).get(1).getValues()[0], "2");
24
22
 
25
- assertEquals(dest.get(1).size(), 2);
26
- assertEquals(dest.get(1).get(0).getName(), "from");
27
- assertEquals(dest.get(1).get(0).getValues()[0], "3");
28
- assertEquals(dest.get(1).get(1).getName(), "to");
29
- assertEquals(dest.get(1).get(1).getValues()[0], "4");
23
+ assertEquals(dest.get(1).size(), 2);
24
+ assertEquals(dest.get(1).get(0).getName(), "from");
25
+ assertEquals(dest.get(1).get(0).getValues()[0], "3");
26
+ assertEquals(dest.get(1).get(1).getName(), "to");
27
+ assertEquals(dest.get(1).get(1).getValues()[0], "4");
30
28
 
31
- assertEquals(dest.get(2).size(), 2);
32
- assertEquals(dest.get(2).get(0).getName(), "from");
33
- assertEquals(dest.get(2).get(0).getValues()[0], "5");
34
- assertEquals(dest.get(2).get(1).getName(), "to");
35
- assertEquals(dest.get(2).get(1).getValues()[0], "6");
36
- }
29
+ assertEquals(dest.get(2).size(), 2);
30
+ assertEquals(dest.get(2).get(0).getName(), "from");
31
+ assertEquals(dest.get(2).get(0).getValues()[0], "5");
32
+ assertEquals(dest.get(2).get(1).getName(), "to");
33
+ assertEquals(dest.get(2).get(1).getValues()[0], "6");
34
+ }
37
35
 
38
- @Test
39
- public void testExpandFromToWithDefault() throws Exception
40
- {
41
- Optional<Integer> nullValue = Optional.absent();
36
+ @Test
37
+ public void testExpandFromToWithDefault() throws Exception {
38
+ Optional<Integer> nullValue = Optional.absent();
42
39
 
43
- List<List<QueryOption.Query>> dest = new PagerOption("from", Optional.of("to"), nullValue, 2, nullValue)
44
- .expand();
45
- assertEquals(dest.size(), 2);
40
+ List<List<QueryOption.Query>> dest =
41
+ new PagerOption("from", Optional.of("to"), nullValue, 2, nullValue).expand();
42
+ assertEquals(dest.size(), 2);
46
43
 
47
- assertEquals(dest.get(0).size(), 2);
48
- assertEquals(dest.get(0).get(0).getName(), "from");
49
- assertEquals(dest.get(0).get(0).getValues()[0], "0");
50
- assertEquals(dest.get(0).get(1).getName(), "to");
51
- assertEquals(dest.get(0).get(1).getValues()[0], "0");
44
+ assertEquals(dest.get(0).size(), 2);
45
+ assertEquals(dest.get(0).get(0).getName(), "from");
46
+ assertEquals(dest.get(0).get(0).getValues()[0], "0");
47
+ assertEquals(dest.get(0).get(1).getName(), "to");
48
+ assertEquals(dest.get(0).get(1).getValues()[0], "0");
52
49
 
53
- assertEquals(dest.get(1).size(), 2);
54
- assertEquals(dest.get(1).get(0).getName(), "from");
55
- assertEquals(dest.get(1).get(0).getValues()[0], "1");
56
- assertEquals(dest.get(1).get(1).getName(), "to");
57
- assertEquals(dest.get(1).get(1).getValues()[0], "1");
58
- }
50
+ assertEquals(dest.get(1).size(), 2);
51
+ assertEquals(dest.get(1).get(0).getName(), "from");
52
+ assertEquals(dest.get(1).get(0).getValues()[0], "1");
53
+ assertEquals(dest.get(1).get(1).getName(), "to");
54
+ assertEquals(dest.get(1).get(1).getValues()[0], "1");
55
+ }
59
56
 
60
- @Test
61
- public void testExpandPagenate() throws Exception
62
- {
63
- Optional<String> nullValue = Optional.absent();
64
- List<List<QueryOption.Query>> dest = new PagerOption("page", nullValue, Optional.of(1), 3,
65
- Optional.of(1)).expand();
66
- assertEquals(dest.size(), 3);
57
+ @Test
58
+ public void testExpandPagenate() throws Exception {
59
+ Optional<String> nullValue = Optional.absent();
60
+ List<List<QueryOption.Query>> dest =
61
+ new PagerOption("page", nullValue, Optional.of(1), 3, Optional.of(1)).expand();
62
+ assertEquals(dest.size(), 3);
67
63
 
68
- assertEquals(dest.get(0).size(), 1);
69
- assertEquals(dest.get(0).get(0).getName(), "page");
70
- assertEquals(dest.get(0).get(0).getValues()[0], "1");
64
+ assertEquals(dest.get(0).size(), 1);
65
+ assertEquals(dest.get(0).get(0).getName(), "page");
66
+ assertEquals(dest.get(0).get(0).getValues()[0], "1");
71
67
 
72
- assertEquals(dest.get(1).size(), 1);
73
- assertEquals(dest.get(1).get(0).getName(), "page");
74
- assertEquals(dest.get(1).get(0).getValues()[0], "2");
68
+ assertEquals(dest.get(1).size(), 1);
69
+ assertEquals(dest.get(1).get(0).getName(), "page");
70
+ assertEquals(dest.get(1).get(0).getValues()[0], "2");
75
71
 
76
- assertEquals(dest.get(2).size(), 1);
77
- assertEquals(dest.get(2).get(0).getName(), "page");
78
- assertEquals(dest.get(2).get(0).getValues()[0], "3");
79
- }
72
+ assertEquals(dest.get(2).size(), 1);
73
+ assertEquals(dest.get(2).get(0).getName(), "page");
74
+ assertEquals(dest.get(2).get(0).getValues()[0], "3");
75
+ }
80
76
  }
@@ -8,102 +8,97 @@ import java.util.List;
8
8
 
9
9
  import static org.junit.Assert.assertEquals;
10
10
 
11
- public class TestParamsOption
12
- {
13
- @Test
14
- public void testUnexpandQueriesSinglePair() throws Exception
15
- {
16
- Optional<List<String>> nullValues = Optional.absent();
17
- QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, false);
18
- QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, false);
19
- ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
20
- Optional<PagerOption> pagerOption = Optional.absent();
21
- List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
22
- assertEquals(dest.size(), 1);
23
- assertEquals(dest.get(0).size(), 2);
24
- assertEquals(dest.get(0).get(0).getName(), "test1");
25
- assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
26
- assertEquals(dest.get(0).get(1).getName(), "test2");
27
- assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
28
- }
11
+ public class TestParamsOption {
12
+ @Test
13
+ public void testUnexpandQueriesSinglePair() throws Exception {
14
+ Optional<List<String>> nullValues = Optional.absent();
15
+ QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, false);
16
+ QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, false);
17
+ ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
18
+ Optional<PagerOption> pagerOption = Optional.absent();
19
+ List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
20
+ assertEquals(dest.size(), 1);
21
+ assertEquals(dest.get(0).size(), 2);
22
+ assertEquals(dest.get(0).get(0).getName(), "test1");
23
+ assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
24
+ assertEquals(dest.get(0).get(1).getName(), "test2");
25
+ assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
26
+ }
29
27
 
30
- @Test
31
- public void testUnexpandQueriesExpandPair() throws Exception
32
- {
33
- Optional<String> nullValue = Optional.absent();
34
- List<String> values1 = Lists.newArrayList("a", "b");
35
- List<String> values2 = Lists.newArrayList("c", "d");
28
+ @Test
29
+ public void testUnexpandQueriesExpandPair() throws Exception {
30
+ Optional<String> nullValue = Optional.absent();
31
+ List<String> values1 = Lists.newArrayList("a", "b");
32
+ List<String> values2 = Lists.newArrayList("c", "d");
36
33
 
37
- QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), false);
38
- QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), false);
34
+ QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), false);
35
+ QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), false);
39
36
 
40
- ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
41
- Optional<PagerOption> pagerOption = Optional.absent();
42
- List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
43
- assertEquals(dest.size(), 1);
44
- assertEquals(dest.get(0).size(), 2);
45
- assertEquals(dest.get(0).get(0).getName(), "test1");
46
- assertEquals(dest.get(0).get(0).getValues()[0], "a");
47
- assertEquals(dest.get(0).get(0).getValues()[1], "b");
48
- assertEquals(dest.get(0).get(1).getName(), "test2");
49
- assertEquals(dest.get(0).get(1).getValues()[0], "c");
50
- assertEquals(dest.get(0).get(1).getValues()[1], "d");
51
- }
37
+ ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
38
+ Optional<PagerOption> pagerOption = Optional.absent();
39
+ List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
40
+ assertEquals(dest.size(), 1);
41
+ assertEquals(dest.get(0).size(), 2);
42
+ assertEquals(dest.get(0).get(0).getName(), "test1");
43
+ assertEquals(dest.get(0).get(0).getValues()[0], "a");
44
+ assertEquals(dest.get(0).get(0).getValues()[1], "b");
45
+ assertEquals(dest.get(0).get(1).getName(), "test2");
46
+ assertEquals(dest.get(0).get(1).getValues()[0], "c");
47
+ assertEquals(dest.get(0).get(1).getValues()[1], "d");
48
+ }
52
49
 
53
- @Test
54
- public void testExpandQueriesSinglePair() throws Exception
55
- {
56
- Optional<List<String>> nullValues = Optional.absent();
57
- QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, true);
58
- QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, true);
59
- ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
60
- Optional<PagerOption> pagerOption = Optional.absent();
61
- List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
62
- assertEquals(dest.size(), 1);
63
- assertEquals(dest.get(0).size(), 2);
64
- assertEquals(dest.get(0).get(0).getName(), "test1");
65
- assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
66
- assertEquals(dest.get(0).get(1).getName(), "test2");
67
- assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
68
- }
50
+ @Test
51
+ public void testExpandQueriesSinglePair() throws Exception {
52
+ Optional<List<String>> nullValues = Optional.absent();
53
+ QueryOption q1 = new QueryOption("test1", Optional.of("awasome1"), nullValues, true);
54
+ QueryOption q2 = new QueryOption("test2", Optional.of("awasome2"), nullValues, true);
55
+ ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
56
+ Optional<PagerOption> pagerOption = Optional.absent();
57
+ List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
58
+ assertEquals(dest.size(), 1);
59
+ assertEquals(dest.get(0).size(), 2);
60
+ assertEquals(dest.get(0).get(0).getName(), "test1");
61
+ assertEquals(dest.get(0).get(0).getValues()[0], "awasome1");
62
+ assertEquals(dest.get(0).get(1).getName(), "test2");
63
+ assertEquals(dest.get(0).get(1).getValues()[0], "awasome2");
64
+ }
69
65
 
70
- @Test
71
- public void testExpandQueriesExpandPair() throws Exception
72
- {
73
- Optional<String> nullValue = Optional.absent();
74
- List<String> values1 = Lists.newArrayList("a", "b");
75
- List<String> values2 = Lists.newArrayList("c", "d");
66
+ @Test
67
+ public void testExpandQueriesExpandPair() throws Exception {
68
+ Optional<String> nullValue = Optional.absent();
69
+ List<String> values1 = Lists.newArrayList("a", "b");
70
+ List<String> values2 = Lists.newArrayList("c", "d");
76
71
 
77
- QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), true);
78
- QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), true);
72
+ QueryOption q1 = new QueryOption("test1", nullValue, Optional.of(values1), true);
73
+ QueryOption q2 = new QueryOption("test2", nullValue, Optional.of(values2), true);
79
74
 
80
- ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
81
- Optional<PagerOption> pagerOption = Optional.absent();
82
- List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
83
- assertEquals(dest.size(), 4);
75
+ ParamsOption paramsOption = new ParamsOption(Lists.newArrayList(q1, q2));
76
+ Optional<PagerOption> pagerOption = Optional.absent();
77
+ List<List<QueryOption.Query>> dest = paramsOption.generateQueries(pagerOption);
78
+ assertEquals(dest.size(), 4);
84
79
 
85
- assertEquals(dest.get(0).size(), 2);
86
- assertEquals(dest.get(0).get(0).getName(), "test1");
87
- assertEquals(dest.get(0).get(0).getValues()[0], "a");
88
- assertEquals(dest.get(0).get(1).getName(), "test2");
89
- assertEquals(dest.get(0).get(1).getValues()[0], "c");
80
+ assertEquals(dest.get(0).size(), 2);
81
+ assertEquals(dest.get(0).get(0).getName(), "test1");
82
+ assertEquals(dest.get(0).get(0).getValues()[0], "a");
83
+ assertEquals(dest.get(0).get(1).getName(), "test2");
84
+ assertEquals(dest.get(0).get(1).getValues()[0], "c");
90
85
 
91
- assertEquals(dest.get(1).size(), 2);
92
- assertEquals(dest.get(1).get(0).getName(), "test1");
93
- assertEquals(dest.get(1).get(0).getValues()[0], "b");
94
- assertEquals(dest.get(1).get(1).getName(), "test2");
95
- assertEquals(dest.get(1).get(1).getValues()[0], "c");
86
+ assertEquals(dest.get(1).size(), 2);
87
+ assertEquals(dest.get(1).get(0).getName(), "test1");
88
+ assertEquals(dest.get(1).get(0).getValues()[0], "b");
89
+ assertEquals(dest.get(1).get(1).getName(), "test2");
90
+ assertEquals(dest.get(1).get(1).getValues()[0], "c");
96
91
 
97
- assertEquals(dest.get(2).size(), 2);
98
- assertEquals(dest.get(2).get(0).getName(), "test1");
99
- assertEquals(dest.get(2).get(0).getValues()[0], "a");
100
- assertEquals(dest.get(2).get(1).getName(), "test2");
101
- assertEquals(dest.get(2).get(1).getValues()[0], "d");
92
+ assertEquals(dest.get(2).size(), 2);
93
+ assertEquals(dest.get(2).get(0).getName(), "test1");
94
+ assertEquals(dest.get(2).get(0).getValues()[0], "a");
95
+ assertEquals(dest.get(2).get(1).getName(), "test2");
96
+ assertEquals(dest.get(2).get(1).getValues()[0], "d");
102
97
 
103
- assertEquals(dest.get(3).size(), 2);
104
- assertEquals(dest.get(3).get(0).getName(), "test1");
105
- assertEquals(dest.get(3).get(0).getValues()[0], "b");
106
- assertEquals(dest.get(3).get(1).getName(), "test2");
107
- assertEquals(dest.get(3).get(1).getValues()[0], "d");
108
- }
98
+ assertEquals(dest.get(3).size(), 2);
99
+ assertEquals(dest.get(3).get(0).getName(), "test1");
100
+ assertEquals(dest.get(3).get(0).getValues()[0], "b");
101
+ assertEquals(dest.get(3).get(1).getName(), "test2");
102
+ assertEquals(dest.get(3).get(1).getValues()[0], "d");
103
+ }
109
104
  }