embulk-input-http 0.0.20 → 0.21.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +3 -3
- data/src/main/java/org/embulk/input/http/BasicAuthOption.java +25 -31
- data/src/main/java/org/embulk/input/http/HttpFileInputPlugin.java +274 -296
- data/src/main/java/org/embulk/input/http/PagerOption.java +73 -76
- data/src/main/java/org/embulk/input/http/ParamsOption.java +57 -66
- data/src/main/java/org/embulk/input/http/QueryOption.java +138 -161
- data/src/main/java/org/embulk/input/http/RetryableHandler.java +57 -63
- data/src/test/java/org/embulk/input/http/TestHttpInputPlugin.java +1 -3
- data/src/test/java/org/embulk/input/http/TestPagerOption.java +55 -59
- data/src/test/java/org/embulk/input/http/TestParamsOption.java +82 -87
- data/src/test/java/org/embulk/input/http/TestQueryOption.java +116 -121
- metadata +3 -3
@@ -7,89 +7,86 @@ import com.google.common.base.Optional;
|
|
7
7
|
import java.util.ArrayList;
|
8
8
|
import java.util.List;
|
9
9
|
|
10
|
-
public class PagerOption
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
private final int step;
|
10
|
+
public class PagerOption {
|
11
|
+
private final String fromParam;
|
12
|
+
private final Optional<String> toParam;
|
13
|
+
private final int start;
|
14
|
+
private final int pages;
|
15
|
+
private final int step;
|
17
16
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
17
|
+
@JsonCreator
|
18
|
+
public PagerOption(
|
19
|
+
@JsonProperty("from_param") String fromParam,
|
20
|
+
@JsonProperty("to_param") Optional<String> toParam,
|
21
|
+
@JsonProperty("start") Optional<Integer> start,
|
22
|
+
@JsonProperty("pages") int pages,
|
23
|
+
@JsonProperty("step") Optional<Integer> step) {
|
24
|
+
this.fromParam = fromParam;
|
25
|
+
this.toParam = toParam;
|
26
|
+
this.start = start.or(0);
|
27
|
+
this.pages = pages;
|
28
|
+
this.step = step.or(1);
|
29
|
+
}
|
31
30
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
queries.add(one);
|
49
|
-
p++;
|
50
|
-
}
|
51
|
-
return queries;
|
31
|
+
public List<List<QueryOption.Query>> expand() {
|
32
|
+
List<List<QueryOption.Query>> queries = new ArrayList<>();
|
33
|
+
int p = 1;
|
34
|
+
int index = start;
|
35
|
+
while (p <= pages) {
|
36
|
+
List<QueryOption.Query> one = new ArrayList<>();
|
37
|
+
one.add(new QueryOption.Query(fromParam, Integer.toString(index)));
|
38
|
+
if (toParam.isPresent()) {
|
39
|
+
int t = index + step - 1;
|
40
|
+
one.add(new QueryOption.Query(toParam.get(), Integer.toString(t)));
|
41
|
+
index = t + 1;
|
42
|
+
} else {
|
43
|
+
index += step;
|
44
|
+
}
|
45
|
+
queries.add(one);
|
46
|
+
p++;
|
52
47
|
}
|
48
|
+
return queries;
|
49
|
+
}
|
53
50
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
}
|
51
|
+
@JsonProperty("from_param")
|
52
|
+
public String getFromParam() {
|
53
|
+
return fromParam;
|
54
|
+
}
|
59
55
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
}
|
56
|
+
@JsonProperty("to_param")
|
57
|
+
public Optional<String> getToParam() {
|
58
|
+
return toParam;
|
59
|
+
}
|
65
60
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
}
|
61
|
+
@JsonProperty("start")
|
62
|
+
public int getStart() {
|
63
|
+
return start;
|
64
|
+
}
|
71
65
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
}
|
66
|
+
@JsonProperty("pages")
|
67
|
+
public int getPages() {
|
68
|
+
return pages;
|
69
|
+
}
|
77
70
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
}
|
71
|
+
@JsonProperty("step")
|
72
|
+
public int getStep() {
|
73
|
+
return step;
|
74
|
+
}
|
83
75
|
|
84
|
-
|
85
|
-
|
86
|
-
{
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
76
|
+
@Override
|
77
|
+
public String toString() {
|
78
|
+
return "PagerOption{"
|
79
|
+
+ "fromParam='"
|
80
|
+
+ fromParam
|
81
|
+
+ '\''
|
82
|
+
+ ", toParam="
|
83
|
+
+ toParam
|
84
|
+
+ ", start="
|
85
|
+
+ start
|
86
|
+
+ ", pages="
|
87
|
+
+ pages
|
88
|
+
+ ", step="
|
89
|
+
+ step
|
90
|
+
+ '}';
|
91
|
+
}
|
95
92
|
}
|
@@ -8,83 +8,74 @@ import com.google.common.base.Optional;
|
|
8
8
|
import java.util.ArrayList;
|
9
9
|
import java.util.List;
|
10
10
|
|
11
|
-
public class ParamsOption
|
12
|
-
|
13
|
-
private final List<QueryOption> queries;
|
11
|
+
public class ParamsOption {
|
12
|
+
private final List<QueryOption> queries;
|
14
13
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
14
|
+
@JsonCreator
|
15
|
+
public ParamsOption(List<QueryOption> queries) {
|
16
|
+
this.queries = queries;
|
17
|
+
}
|
18
|
+
|
19
|
+
@JsonValue
|
20
|
+
public List<QueryOption> getQueries() {
|
21
|
+
return queries;
|
22
|
+
}
|
20
23
|
|
21
|
-
|
22
|
-
|
23
|
-
{
|
24
|
-
|
24
|
+
public List<List<QueryOption.Query>> generateQueries(Optional<PagerOption> pagerOption) {
|
25
|
+
List<List<QueryOption.Query>> base = new ArrayList<>(queries.size());
|
26
|
+
for (QueryOption p : queries) {
|
27
|
+
base.add(p.expand());
|
25
28
|
}
|
26
29
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
for (QueryOption p : queries) {
|
31
|
-
base.add(p.expand());
|
32
|
-
}
|
30
|
+
int productSize = 1;
|
31
|
+
int baseSize = base.size();
|
32
|
+
for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++) {}
|
33
33
|
|
34
|
-
|
35
|
-
|
36
|
-
|
34
|
+
List<List<QueryOption.Query>> expands = new ArrayList<>(productSize);
|
35
|
+
for (int i = 0; i < productSize; i++) {
|
36
|
+
int j = 1;
|
37
|
+
List<QueryOption.Query> one = new ArrayList<>();
|
38
|
+
for (List<QueryOption.Query> list : base) {
|
39
|
+
QueryOption.Query pc = list.get((i / j) % list.size());
|
40
|
+
one.add(pc);
|
41
|
+
j *= list.size();
|
42
|
+
}
|
43
|
+
if (pagerOption.isPresent()) {
|
44
|
+
for (List<QueryOption.Query> q : pagerOption.get().expand()) {
|
45
|
+
expands.add(copyAndConcat(one, q));
|
37
46
|
}
|
47
|
+
} else {
|
48
|
+
expands.add(one);
|
49
|
+
}
|
50
|
+
}
|
38
51
|
|
39
|
-
|
40
|
-
|
41
|
-
int j = 1;
|
42
|
-
List<QueryOption.Query> one = new ArrayList<>();
|
43
|
-
for (List<QueryOption.Query> list : base) {
|
44
|
-
QueryOption.Query pc = list.get((i / j) % list.size());
|
45
|
-
one.add(pc);
|
46
|
-
j *= list.size();
|
47
|
-
}
|
48
|
-
if (pagerOption.isPresent()) {
|
49
|
-
for (List<QueryOption.Query> q : pagerOption.get().expand()) {
|
50
|
-
expands.add(copyAndConcat(one, q));
|
51
|
-
}
|
52
|
-
}
|
53
|
-
else {
|
54
|
-
expands.add(one);
|
55
|
-
}
|
56
|
-
}
|
52
|
+
return expands;
|
53
|
+
}
|
57
54
|
|
58
|
-
|
55
|
+
@Override
|
56
|
+
public boolean equals(Object obj) {
|
57
|
+
if (this == obj) {
|
58
|
+
return true;
|
59
59
|
}
|
60
|
-
|
61
|
-
|
62
|
-
public boolean equals(Object obj)
|
63
|
-
{
|
64
|
-
if (this == obj) {
|
65
|
-
return true;
|
66
|
-
}
|
67
|
-
if (!(obj instanceof ParamsOption)) {
|
68
|
-
return false;
|
69
|
-
}
|
70
|
-
ParamsOption other = (ParamsOption) obj;
|
71
|
-
return Objects.equal(queries, other.queries);
|
60
|
+
if (!(obj instanceof ParamsOption)) {
|
61
|
+
return false;
|
72
62
|
}
|
63
|
+
ParamsOption other = (ParamsOption) obj;
|
64
|
+
return Objects.equal(queries, other.queries);
|
65
|
+
}
|
73
66
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
}
|
67
|
+
@Override
|
68
|
+
public int hashCode() {
|
69
|
+
return Objects.hashCode(queries);
|
70
|
+
}
|
79
71
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
}
|
87
|
-
}
|
88
|
-
return dest;
|
72
|
+
private List<QueryOption.Query> copyAndConcat(List<QueryOption.Query>... srcs) {
|
73
|
+
List<QueryOption.Query> dest = new ArrayList<>();
|
74
|
+
for (List<QueryOption.Query> src : srcs) {
|
75
|
+
for (QueryOption.Query q : src) {
|
76
|
+
dest.add(q.copy());
|
77
|
+
}
|
89
78
|
}
|
79
|
+
return dest;
|
80
|
+
}
|
90
81
|
}
|
@@ -9,184 +9,161 @@ import java.util.ArrayList;
|
|
9
9
|
import java.util.Arrays;
|
10
10
|
import java.util.List;
|
11
11
|
|
12
|
-
public class QueryOption
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
{
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
for (String s : expanded) {
|
39
|
-
dest.add(new Query(name, s));
|
40
|
-
}
|
41
|
-
}
|
42
|
-
else {
|
43
|
-
dest = new ArrayList<>(1);
|
44
|
-
dest.add(new Query(name, value.get()));
|
45
|
-
}
|
12
|
+
public class QueryOption {
|
13
|
+
private final String name;
|
14
|
+
private final Optional<String> value;
|
15
|
+
private final Optional<List<String>> values;
|
16
|
+
private final boolean expand;
|
17
|
+
|
18
|
+
@JsonCreator
|
19
|
+
public QueryOption(
|
20
|
+
@JsonProperty("name") String name,
|
21
|
+
@JsonProperty("value") Optional<String> value,
|
22
|
+
@JsonProperty("values") Optional<List<String>> values,
|
23
|
+
@JsonProperty("expand") boolean expand) {
|
24
|
+
this.name = name;
|
25
|
+
this.value = value;
|
26
|
+
this.values = values;
|
27
|
+
this.expand = expand;
|
28
|
+
}
|
29
|
+
|
30
|
+
public List<Query> expand() {
|
31
|
+
List<Query> dest;
|
32
|
+
if (value.isPresent()) {
|
33
|
+
if (expand) {
|
34
|
+
List<String> expanded = BraceExpansion.expand(value.get());
|
35
|
+
dest = new ArrayList<>(expanded.size());
|
36
|
+
for (String s : expanded) {
|
37
|
+
dest.add(new Query(name, s));
|
46
38
|
}
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
final String[] valueArr = values.get().toArray(new String[values.get().size()]);
|
57
|
-
dest.add(new Query(name, valueArr));
|
58
|
-
}
|
39
|
+
} else {
|
40
|
+
dest = new ArrayList<>(1);
|
41
|
+
dest.add(new Query(name, value.get()));
|
42
|
+
}
|
43
|
+
} else if (values.isPresent()) {
|
44
|
+
if (expand) {
|
45
|
+
dest = new ArrayList<>(values.get().size());
|
46
|
+
for (String s : values.get()) {
|
47
|
+
dest.add(new Query(name, s));
|
59
48
|
}
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
49
|
+
} else {
|
50
|
+
dest = new ArrayList<>(1);
|
51
|
+
final String[] valueArr = values.get().toArray(new String[values.get().size()]);
|
52
|
+
dest.add(new Query(name, valueArr));
|
53
|
+
}
|
54
|
+
} else {
|
55
|
+
throw new IllegalArgumentException("value or values must be specified to 'params'");
|
64
56
|
}
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
57
|
+
return dest;
|
58
|
+
}
|
59
|
+
|
60
|
+
@JsonProperty("name")
|
61
|
+
public String getName() {
|
62
|
+
return name;
|
63
|
+
}
|
64
|
+
|
65
|
+
@JsonProperty("value")
|
66
|
+
public Optional<String> getValue() {
|
67
|
+
return value;
|
68
|
+
}
|
69
|
+
|
70
|
+
@JsonProperty("expand")
|
71
|
+
public boolean isExpand() {
|
72
|
+
return expand;
|
73
|
+
}
|
74
|
+
|
75
|
+
@Override
|
76
|
+
public boolean equals(Object obj) {
|
77
|
+
if (this == obj) {
|
78
|
+
return true;
|
70
79
|
}
|
71
|
-
|
72
|
-
|
73
|
-
public Optional<String> getValue()
|
74
|
-
{
|
75
|
-
return value;
|
80
|
+
if (!(obj instanceof QueryOption)) {
|
81
|
+
return false;
|
76
82
|
}
|
83
|
+
QueryOption other = (QueryOption) obj;
|
84
|
+
return Objects.equal(this.name, other.name)
|
85
|
+
&& Objects.equal(value, other.value)
|
86
|
+
&& Objects.equal(expand, other.expand);
|
87
|
+
}
|
88
|
+
|
89
|
+
@Override
|
90
|
+
public int hashCode() {
|
91
|
+
return Objects.hashCode(name, value, expand);
|
92
|
+
}
|
93
|
+
|
94
|
+
@Override
|
95
|
+
public String toString() {
|
96
|
+
return String.format("ParameterConfig[%s, %s, %s]", getName(), getValue(), isExpand());
|
97
|
+
}
|
98
|
+
|
99
|
+
public static class Query {
|
100
|
+
private final String name;
|
101
|
+
private final String[] values;
|
77
102
|
|
78
|
-
@JsonProperty("
|
79
|
-
|
80
|
-
|
81
|
-
return expand;
|
103
|
+
public Query(@JsonProperty("name") String name, @JsonProperty("values") String... values) {
|
104
|
+
this.name = name;
|
105
|
+
this.values = values;
|
82
106
|
}
|
83
107
|
|
84
|
-
|
85
|
-
|
86
|
-
{
|
87
|
-
if (this == obj) {
|
88
|
-
return true;
|
89
|
-
}
|
90
|
-
if (!(obj instanceof QueryOption)) {
|
91
|
-
return false;
|
92
|
-
}
|
93
|
-
QueryOption other = (QueryOption) obj;
|
94
|
-
return Objects.equal(this.name, other.name) &&
|
95
|
-
Objects.equal(value, other.value) &&
|
96
|
-
Objects.equal(expand, other.expand);
|
108
|
+
public String getName() {
|
109
|
+
return name;
|
97
110
|
}
|
98
111
|
|
99
|
-
|
100
|
-
|
101
|
-
{
|
102
|
-
return Objects.hashCode(name, value, expand);
|
112
|
+
public String[] getValues() {
|
113
|
+
return values;
|
103
114
|
}
|
104
115
|
|
105
|
-
|
106
|
-
|
107
|
-
{
|
108
|
-
return String.format("ParameterConfig[%s, %s, %s]",
|
109
|
-
getName(), getValue(), isExpand());
|
116
|
+
public Query copy() {
|
117
|
+
return new Query(this.name, Arrays.copyOf(this.values, this.values.length));
|
110
118
|
}
|
119
|
+
}
|
111
120
|
|
112
|
-
|
113
|
-
{
|
114
|
-
|
115
|
-
private final String[] values;
|
116
|
-
|
117
|
-
public Query(@JsonProperty("name") String name,
|
118
|
-
@JsonProperty("values") String... values)
|
119
|
-
{
|
120
|
-
this.name = name;
|
121
|
-
this.values = values;
|
122
|
-
}
|
123
|
-
|
124
|
-
public String getName()
|
125
|
-
{
|
126
|
-
return name;
|
127
|
-
}
|
128
|
-
|
129
|
-
public String[] getValues()
|
130
|
-
{
|
131
|
-
return values;
|
132
|
-
}
|
133
|
-
|
134
|
-
public Query copy()
|
135
|
-
{
|
136
|
-
return new Query(this.name, Arrays.copyOf(this.values, this.values.length));
|
137
|
-
}
|
121
|
+
private static class BraceExpansion {
|
122
|
+
public static List<String> expand(String s) {
|
123
|
+
return expandRecursive("", s, "", new ArrayList<String>());
|
138
124
|
}
|
139
125
|
|
140
|
-
private static
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
126
|
+
private static List<String> expandRecursive(
|
127
|
+
String prefix, String s, String suffix, List<String> dest) {
|
128
|
+
// used the code below as reference.
|
129
|
+
// http://rosettacode.org/wiki/Brace_expansion#Java
|
130
|
+
int i1 = -1;
|
131
|
+
int i2 = 0;
|
132
|
+
String noEscape = s.replaceAll("([\\\\]{2}|[\\\\][,}{])", " ");
|
133
|
+
StringBuilder sb = null;
|
134
|
+
|
135
|
+
outer:
|
136
|
+
while ((i1 = noEscape.indexOf('{', i1 + 1)) != -1) {
|
137
|
+
i2 = i1 + 1;
|
138
|
+
sb = new StringBuilder(s);
|
139
|
+
for (int depth = 1; i2 < s.length() && depth > 0; i2++) {
|
140
|
+
char c = noEscape.charAt(i2);
|
141
|
+
depth = (c == '{') ? ++depth : depth;
|
142
|
+
depth = (c == '}') ? --depth : depth;
|
143
|
+
if (c == ',' && depth == 1) {
|
144
|
+
sb.setCharAt(i2, '\u0000');
|
145
|
+
} else if (c == '}' && depth == 0 && sb.indexOf("\u0000") != -1) {
|
146
|
+
break outer;
|
147
|
+
}
|
145
148
|
}
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
{
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
for (int depth = 1; i2 < s.length() && depth > 0; i2++) {
|
162
|
-
char c = noEscape.charAt(i2);
|
163
|
-
depth = (c == '{') ? ++depth : depth;
|
164
|
-
depth = (c == '}') ? --depth : depth;
|
165
|
-
if (c == ',' && depth == 1) {
|
166
|
-
sb.setCharAt(i2, '\u0000');
|
167
|
-
}
|
168
|
-
else if (c == '}' && depth == 0 && sb.indexOf("\u0000") != -1) {
|
169
|
-
break outer;
|
170
|
-
}
|
171
|
-
}
|
172
|
-
}
|
173
|
-
|
174
|
-
if (i1 == -1) {
|
175
|
-
if (suffix.length() > 0) {
|
176
|
-
expandRecursive(prefix + s, suffix, "", dest);
|
177
|
-
}
|
178
|
-
else {
|
179
|
-
final String out = String.format("%s%s%s", prefix, s, suffix).
|
180
|
-
replaceAll("[\\\\]{2}", "\\").replaceAll("[\\\\]([,}{])", "$1");
|
181
|
-
dest.add(out);
|
182
|
-
}
|
183
|
-
}
|
184
|
-
else {
|
185
|
-
for (String m : sb.substring(i1 + 1, i2).split("\u0000", -1)) {
|
186
|
-
expandRecursive(prefix + s.substring(0, i1), m, s.substring(i2 + 1) + suffix, dest);
|
187
|
-
}
|
188
|
-
}
|
189
|
-
return dest;
|
149
|
+
}
|
150
|
+
|
151
|
+
if (i1 == -1) {
|
152
|
+
if (suffix.length() > 0) {
|
153
|
+
expandRecursive(prefix + s, suffix, "", dest);
|
154
|
+
} else {
|
155
|
+
final String out =
|
156
|
+
String.format("%s%s%s", prefix, s, suffix)
|
157
|
+
.replaceAll("[\\\\]{2}", "\\")
|
158
|
+
.replaceAll("[\\\\]([,}{])", "$1");
|
159
|
+
dest.add(out);
|
160
|
+
}
|
161
|
+
} else {
|
162
|
+
for (String m : sb.substring(i1 + 1, i2).split("\u0000", -1)) {
|
163
|
+
expandRecursive(prefix + s.substring(0, i1), m, s.substring(i2 + 1) + suffix, dest);
|
190
164
|
}
|
165
|
+
}
|
166
|
+
return dest;
|
191
167
|
}
|
168
|
+
}
|
192
169
|
}
|