embulk-input-http 0.0.20 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +3 -3
- data/src/main/java/org/embulk/input/http/BasicAuthOption.java +25 -31
- data/src/main/java/org/embulk/input/http/HttpFileInputPlugin.java +274 -296
- data/src/main/java/org/embulk/input/http/PagerOption.java +73 -76
- data/src/main/java/org/embulk/input/http/ParamsOption.java +57 -66
- data/src/main/java/org/embulk/input/http/QueryOption.java +138 -161
- data/src/main/java/org/embulk/input/http/RetryableHandler.java +57 -63
- data/src/test/java/org/embulk/input/http/TestHttpInputPlugin.java +1 -3
- data/src/test/java/org/embulk/input/http/TestPagerOption.java +55 -59
- data/src/test/java/org/embulk/input/http/TestParamsOption.java +82 -87
- data/src/test/java/org/embulk/input/http/TestQueryOption.java +116 -121
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f4e401d8cc7ca747a5b6c54a66b2729ae9aab821
|
4
|
+
data.tar.gz: aa33a85c06989405527ce926c79f8ceb22bc2eba
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 779faf323699e45d19dc490dbb48840fbc0354e73b83a37c6b4dcbbb54b890ab39cf1da4b4089f44a47418fd0686c169c974bbdb4534906e5de34ae3b1862837
|
7
|
+
data.tar.gz: 0d20d2060aff2cfddccf3394c5ce0817e4b389146fb468812e86962f75e750b3c3d8d176b0ae025e922739709caca0fd69d78a23c0ccb1c75307a323c6ac4b5d
|
data/build.gradle
CHANGED
@@ -13,14 +13,14 @@ configurations {
|
|
13
13
|
provided
|
14
14
|
}
|
15
15
|
|
16
|
-
version = "0.0
|
16
|
+
version = "0.21.0"
|
17
17
|
|
18
18
|
sourceCompatibility = 1.7
|
19
19
|
targetCompatibility = 1.7
|
20
20
|
|
21
21
|
dependencies {
|
22
|
-
compile "org.embulk:embulk-core:0.9.
|
23
|
-
provided "org.embulk:embulk-core:0.9.
|
22
|
+
compile "org.embulk:embulk-core:0.9.14"
|
23
|
+
provided "org.embulk:embulk-core:0.9.14"
|
24
24
|
compile "org.apache.httpcomponents:httpclient:4.5.5"
|
25
25
|
compile "commons-io:commons-io:2.6"
|
26
26
|
testCompile "junit:junit:4.+"
|
@@ -4,40 +4,34 @@ import com.fasterxml.jackson.annotation.JsonCreator;
|
|
4
4
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
5
|
import com.google.common.base.Objects;
|
6
6
|
|
7
|
-
public class BasicAuthOption
|
8
|
-
|
9
|
-
|
10
|
-
private final String password;
|
7
|
+
public class BasicAuthOption {
|
8
|
+
private final String user;
|
9
|
+
private final String password;
|
11
10
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
}
|
11
|
+
@JsonCreator
|
12
|
+
public BasicAuthOption(
|
13
|
+
@JsonProperty("user") String user, @JsonProperty("password") String password) {
|
14
|
+
this.user = user;
|
15
|
+
this.password = password;
|
16
|
+
}
|
19
17
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
}
|
18
|
+
@JsonProperty("user")
|
19
|
+
public String getUser() {
|
20
|
+
return user;
|
21
|
+
}
|
25
22
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
}
|
23
|
+
@JsonProperty("password")
|
24
|
+
public String getPassword() {
|
25
|
+
return password;
|
26
|
+
}
|
31
27
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
}
|
28
|
+
@Override
|
29
|
+
public int hashCode() {
|
30
|
+
return Objects.hashCode(user, password);
|
31
|
+
}
|
37
32
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
}
|
33
|
+
@Override
|
34
|
+
public String toString() {
|
35
|
+
return String.format("BasicAuthOption[%s, %s]", getUser(), getPassword());
|
36
|
+
}
|
43
37
|
}
|
@@ -52,352 +52,330 @@ import java.util.Map;
|
|
52
52
|
|
53
53
|
import static java.lang.String.format;
|
54
54
|
|
55
|
-
public class HttpFileInputPlugin implements FileInputPlugin
|
56
|
-
|
57
|
-
|
55
|
+
public class HttpFileInputPlugin implements FileInputPlugin {
|
56
|
+
private final Logger logger = Exec.getLogger(getClass());
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
|
60
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
61
|
+
|
62
|
+
final int tasks;
|
63
|
+
if (task.getParams().isPresent()) {
|
64
|
+
List<List<QueryOption.Query>> queries =
|
65
|
+
task.getParams().get().generateQueries(task.getPager());
|
66
|
+
task.setQueries(queries);
|
67
|
+
tasks = queries.size();
|
68
|
+
} else if (task.getPager().isPresent()) {
|
69
|
+
List<List<QueryOption.Query>> queries = task.getPager().get().expand();
|
70
|
+
task.setQueries(queries);
|
71
|
+
tasks = queries.size();
|
72
|
+
} else {
|
73
|
+
task.setQueries(Lists.<List<QueryOption.Query>>newArrayList());
|
74
|
+
task.setRequestInterval(0);
|
75
|
+
tasks = 1;
|
76
|
+
}
|
58
77
|
|
59
|
-
|
60
|
-
{
|
61
|
-
@Config("url")
|
62
|
-
String getUrl();
|
78
|
+
task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
|
63
79
|
|
64
|
-
|
65
|
-
|
66
|
-
String getCharset();
|
80
|
+
return resume(task.dump(), tasks, control);
|
81
|
+
}
|
67
82
|
|
68
|
-
|
69
|
-
|
70
|
-
|
83
|
+
@Override
|
84
|
+
public ConfigDiff resume(TaskSource taskSource, int taskCount, FileInputPlugin.Control control) {
|
85
|
+
control.run(taskSource, taskCount);
|
86
|
+
return Exec.newConfigDiff();
|
87
|
+
}
|
71
88
|
|
72
|
-
|
73
|
-
|
74
|
-
String getUserAgent();
|
89
|
+
@Override
|
90
|
+
public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {}
|
75
91
|
|
76
|
-
|
77
|
-
|
78
|
-
|
92
|
+
@Override
|
93
|
+
public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
|
94
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
79
95
|
|
80
|
-
|
81
|
-
|
82
|
-
|
96
|
+
HttpRequestBase request;
|
97
|
+
try {
|
98
|
+
request = makeRequest(task, taskIndex);
|
99
|
+
} catch (URISyntaxException | UnsupportedEncodingException e) {
|
100
|
+
throw Throwables.propagate(e);
|
101
|
+
}
|
83
102
|
|
84
|
-
|
85
|
-
|
86
|
-
|
103
|
+
HttpClientBuilder builder =
|
104
|
+
HttpClientBuilder.create()
|
105
|
+
.disableAutomaticRetries()
|
106
|
+
.setDefaultRequestConfig(makeRequestConfig(task))
|
107
|
+
.setDefaultHeaders(makeHeaders(task));
|
87
108
|
|
88
|
-
|
89
|
-
|
90
|
-
|
109
|
+
if (task.getBasicAuth().isPresent()) {
|
110
|
+
builder.setDefaultCredentialsProvider(
|
111
|
+
makeCredentialsProvider(task.getBasicAuth().get(), request));
|
112
|
+
}
|
91
113
|
|
92
|
-
|
93
|
-
|
94
|
-
|
114
|
+
HttpClient client = builder.build();
|
115
|
+
|
116
|
+
logger.info(
|
117
|
+
format(
|
118
|
+
Locale.ENGLISH,
|
119
|
+
"%s \"%s\"",
|
120
|
+
task.getMethod().toUpperCase(),
|
121
|
+
request.getURI().toString()));
|
122
|
+
|
123
|
+
RetryableHandler retryable = new RetryableHandler(client, request);
|
124
|
+
long startTimeMills = System.currentTimeMillis();
|
125
|
+
try {
|
126
|
+
RetryExecutor.retryExecutor()
|
127
|
+
.withRetryLimit(task.getMaxRetries())
|
128
|
+
.withInitialRetryWait(task.getRetryInterval())
|
129
|
+
.withMaxRetryWait(30 * 60 * 1000)
|
130
|
+
.runInterruptible(retryable);
|
131
|
+
|
132
|
+
InputStream stream = retryable.getResponse().getEntity().getContent();
|
133
|
+
if (!task.getInputDirect()) {
|
134
|
+
stream = copyToFile(stream);
|
135
|
+
}
|
136
|
+
|
137
|
+
PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
|
138
|
+
stream = null;
|
139
|
+
return input;
|
140
|
+
} catch (Exception e) {
|
141
|
+
throw Throwables.propagate(e);
|
142
|
+
}
|
143
|
+
}
|
95
144
|
|
96
|
-
|
145
|
+
private InputStream copyToFile(InputStream input) throws IOException {
|
146
|
+
File tmpfile = Files.createTempFile("embulk-input-http.", ".tmp").toFile();
|
147
|
+
tmpfile.deleteOnExit();
|
97
148
|
|
98
|
-
|
99
|
-
|
100
|
-
|
149
|
+
try (FileOutputStream output = new FileOutputStream(tmpfile)) {
|
150
|
+
logger.info(format(Locale.ENGLISH, "Writing response to %s", tmpfile));
|
151
|
+
IOUtils.copy(input, output);
|
152
|
+
} finally {
|
153
|
+
input.close();
|
154
|
+
}
|
101
155
|
|
102
|
-
|
103
|
-
|
104
|
-
|
156
|
+
return new FileInputStream(tmpfile);
|
157
|
+
}
|
158
|
+
|
159
|
+
private CredentialsProvider makeCredentialsProvider(
|
160
|
+
BasicAuthOption basicAuth, HttpRequestBase request) {
|
161
|
+
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
162
|
+
final AuthScope authScope =
|
163
|
+
new AuthScope(request.getURI().getHost(), request.getURI().getPort());
|
164
|
+
credentialsProvider.setCredentials(
|
165
|
+
authScope, new UsernamePasswordCredentials(basicAuth.getUser(), basicAuth.getPassword()));
|
166
|
+
return credentialsProvider;
|
167
|
+
}
|
168
|
+
|
169
|
+
private HttpRequestBase makeRequest(PluginTask task, int taskIndex)
|
170
|
+
throws URISyntaxException, UnsupportedEncodingException {
|
171
|
+
final List<QueryOption.Query> queries =
|
172
|
+
(task.getQueries().isEmpty()) ? null : task.getQueries().get(taskIndex);
|
173
|
+
if (task.getHttpMethod() == HttpMethod.GET) {
|
174
|
+
HttpGet request = new HttpGet(task.getUrl());
|
175
|
+
if (queries != null) {
|
176
|
+
URIBuilder builder = new URIBuilder(request.getURI());
|
177
|
+
for (QueryOption.Query q : queries) {
|
178
|
+
for (String v : q.getValues()) {
|
179
|
+
builder.addParameter(q.getName(), v);
|
180
|
+
}
|
181
|
+
}
|
182
|
+
request.setURI(builder.build());
|
183
|
+
}
|
184
|
+
return request;
|
185
|
+
}
|
186
|
+
if (task.getHttpMethod() == HttpMethod.POST) {
|
187
|
+
HttpPost request = new HttpPost(task.getUrl());
|
188
|
+
if (queries != null) {
|
189
|
+
List<NameValuePair> pairs = new ArrayList<>();
|
190
|
+
for (QueryOption.Query q : queries) {
|
191
|
+
for (String v : q.getValues()) {
|
192
|
+
pairs.add(new BasicNameValuePair(q.getName(), v));
|
193
|
+
}
|
194
|
+
}
|
195
|
+
request.setEntity(new UrlEncodedFormEntity(pairs));
|
196
|
+
} else if (task.getRequestBody().isPresent()) {
|
197
|
+
logger.info(new StringEntity(task.getRequestBody().get()).toString());
|
198
|
+
request.setEntity(new StringEntity(task.getRequestBody().get()));
|
199
|
+
}
|
200
|
+
return request;
|
201
|
+
}
|
202
|
+
throw new IllegalArgumentException(
|
203
|
+
String.format("Unsupported http method %s", task.getMethod()));
|
204
|
+
}
|
205
|
+
|
206
|
+
private List<Header> makeHeaders(PluginTask task) {
|
207
|
+
List<Header> headers = new ArrayList<>();
|
208
|
+
headers.add(new BasicHeader("Accept", "*/*"));
|
209
|
+
headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
|
210
|
+
headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
|
211
|
+
headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
|
212
|
+
headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
|
213
|
+
for (Map.Entry<String, String> entry : task.getRequestHeaders().entrySet()) {
|
214
|
+
headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
|
215
|
+
}
|
216
|
+
return headers;
|
217
|
+
}
|
105
218
|
|
106
|
-
|
107
|
-
|
108
|
-
|
219
|
+
private RequestConfig makeRequestConfig(PluginTask task) {
|
220
|
+
return RequestConfig.custom()
|
221
|
+
.setCircularRedirectsAllowed(true)
|
222
|
+
.setMaxRedirects(10)
|
223
|
+
.setRedirectsEnabled(true)
|
224
|
+
.setConnectTimeout(task.getOpenTimeout())
|
225
|
+
.setSocketTimeout(task.getReadTimeout())
|
226
|
+
.build();
|
227
|
+
}
|
109
228
|
|
110
|
-
|
111
|
-
|
112
|
-
|
229
|
+
public enum HttpMethod {
|
230
|
+
POST,
|
231
|
+
GET
|
232
|
+
}
|
113
233
|
|
114
|
-
|
115
|
-
|
116
|
-
|
234
|
+
public interface PluginTask extends Task {
|
235
|
+
@Config("url")
|
236
|
+
String getUrl();
|
117
237
|
|
118
|
-
|
119
|
-
|
120
|
-
|
238
|
+
@Config("charset")
|
239
|
+
@ConfigDefault("\"utf-8\"")
|
240
|
+
String getCharset();
|
121
241
|
|
122
|
-
|
123
|
-
|
124
|
-
|
242
|
+
@Config("method")
|
243
|
+
@ConfigDefault("\"get\"")
|
244
|
+
String getMethod();
|
125
245
|
|
126
|
-
|
127
|
-
|
246
|
+
@Config("user_agent")
|
247
|
+
@ConfigDefault("\"Embulk::Input::HttpFileInputPlugin\"")
|
248
|
+
String getUserAgent();
|
128
249
|
|
129
|
-
|
250
|
+
@Config("open_timeout")
|
251
|
+
@ConfigDefault("2000")
|
252
|
+
int getOpenTimeout();
|
130
253
|
|
131
|
-
|
254
|
+
@Config("read_timeout")
|
255
|
+
@ConfigDefault("10000")
|
256
|
+
int getReadTimeout();
|
132
257
|
|
133
|
-
|
258
|
+
@Config("max_retries")
|
259
|
+
@ConfigDefault("5")
|
260
|
+
int getMaxRetries();
|
134
261
|
|
135
|
-
|
136
|
-
|
262
|
+
@Config("retry_interval")
|
263
|
+
@ConfigDefault("10000")
|
264
|
+
int getRetryInterval();
|
137
265
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
GET
|
142
|
-
}
|
266
|
+
@Config("request_interval")
|
267
|
+
@ConfigDefault("0")
|
268
|
+
int getRequestInterval();
|
143
269
|
|
144
|
-
|
145
|
-
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
146
|
-
{
|
147
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
148
|
-
|
149
|
-
final int tasks;
|
150
|
-
if (task.getParams().isPresent()) {
|
151
|
-
List<List<QueryOption.Query>> queries = task.getParams().get().generateQueries(task.getPager());
|
152
|
-
task.setQueries(queries);
|
153
|
-
tasks = queries.size();
|
154
|
-
}
|
155
|
-
else if (task.getPager().isPresent()) {
|
156
|
-
List<List<QueryOption.Query>> queries = task.getPager().get().expand();
|
157
|
-
task.setQueries(queries);
|
158
|
-
tasks = queries.size();
|
159
|
-
}
|
160
|
-
else {
|
161
|
-
task.setQueries(Lists.<List<QueryOption.Query>>newArrayList());
|
162
|
-
task.setRequestInterval(0);
|
163
|
-
tasks = 1;
|
164
|
-
}
|
270
|
+
void setRequestInterval(int requestInterval);
|
165
271
|
|
166
|
-
|
272
|
+
@Config("interval_includes_response_time")
|
273
|
+
@ConfigDefault("null")
|
274
|
+
boolean getIntervalIncludesResponseTime();
|
167
275
|
|
168
|
-
|
169
|
-
|
276
|
+
@Config("input_direct")
|
277
|
+
@ConfigDefault("true")
|
278
|
+
boolean getInputDirect();
|
170
279
|
|
171
|
-
@
|
172
|
-
|
173
|
-
|
174
|
-
control.run(taskSource, taskCount);
|
175
|
-
return Exec.newConfigDiff();
|
176
|
-
}
|
280
|
+
@Config("params")
|
281
|
+
@ConfigDefault("null")
|
282
|
+
Optional<ParamsOption> getParams();
|
177
283
|
|
178
|
-
@
|
179
|
-
|
180
|
-
|
181
|
-
}
|
284
|
+
@Config("request_body")
|
285
|
+
@ConfigDefault("null")
|
286
|
+
Optional<String> getRequestBody();
|
182
287
|
|
183
|
-
@
|
184
|
-
|
185
|
-
|
186
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
288
|
+
@Config("basic_auth")
|
289
|
+
@ConfigDefault("null")
|
290
|
+
Optional<BasicAuthOption> getBasicAuth();
|
187
291
|
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
}
|
192
|
-
catch (URISyntaxException | UnsupportedEncodingException e) {
|
193
|
-
throw Throwables.propagate(e);
|
194
|
-
}
|
292
|
+
@Config("pager")
|
293
|
+
@ConfigDefault("null")
|
294
|
+
Optional<PagerOption> getPager();
|
195
295
|
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
.setDefaultHeaders(makeHeaders(task));
|
296
|
+
@Config("request_headers")
|
297
|
+
@ConfigDefault("{}")
|
298
|
+
Map<String, String> getRequestHeaders();
|
200
299
|
|
201
|
-
|
202
|
-
|
203
|
-
}
|
300
|
+
@ConfigInject
|
301
|
+
BufferAllocator getBufferAllocator();
|
204
302
|
|
205
|
-
|
303
|
+
List<List<QueryOption.Query>> getQueries();
|
206
304
|
|
207
|
-
|
305
|
+
void setQueries(List<List<QueryOption.Query>> queries);
|
208
306
|
|
209
|
-
|
210
|
-
long startTimeMills = System.currentTimeMillis();
|
211
|
-
try {
|
212
|
-
RetryExecutor.retryExecutor().
|
213
|
-
withRetryLimit(task.getMaxRetries()).
|
214
|
-
withInitialRetryWait(task.getRetryInterval()).
|
215
|
-
withMaxRetryWait(30 * 60 * 1000).
|
216
|
-
runInterruptible(retryable);
|
217
|
-
|
218
|
-
InputStream stream = retryable.getResponse().getEntity().getContent();
|
219
|
-
if (!task.getInputDirect()) {
|
220
|
-
stream = copyToFile(stream);
|
221
|
-
}
|
222
|
-
|
223
|
-
PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
|
224
|
-
stream = null;
|
225
|
-
return input;
|
226
|
-
}
|
227
|
-
catch (Exception e) {
|
228
|
-
throw Throwables.propagate(e);
|
229
|
-
}
|
230
|
-
}
|
307
|
+
HttpMethod getHttpMethod();
|
231
308
|
|
232
|
-
|
233
|
-
|
234
|
-
{
|
235
|
-
File tmpfile = Files.createTempFile("embulk-input-http.", ".tmp").toFile();
|
236
|
-
tmpfile.deleteOnExit();
|
237
|
-
|
238
|
-
try (FileOutputStream output = new FileOutputStream(tmpfile)) {
|
239
|
-
logger.info(format(Locale.ENGLISH, "Writing response to %s", tmpfile));
|
240
|
-
IOUtils.copy(input, output);
|
241
|
-
} finally {
|
242
|
-
input.close();
|
243
|
-
}
|
309
|
+
void setHttpMethod(HttpMethod httpMethod);
|
310
|
+
}
|
244
311
|
|
245
|
-
|
246
|
-
|
312
|
+
public static class PluginFileInput extends InputStreamFileInput
|
313
|
+
implements TransactionalFileInput {
|
314
|
+
private final Logger logger = Exec.getLogger(getClass());
|
247
315
|
|
248
|
-
private
|
249
|
-
|
250
|
-
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
251
|
-
final AuthScope authScope = new AuthScope(request.getURI().getHost(),
|
252
|
-
request.getURI().getPort());
|
253
|
-
credentialsProvider.setCredentials(authScope,
|
254
|
-
new UsernamePasswordCredentials(basicAuth.getUser(), basicAuth.getPassword()));
|
255
|
-
return credentialsProvider;
|
256
|
-
}
|
316
|
+
private final long startTimeMills;
|
317
|
+
private final PluginTask task;
|
257
318
|
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
null : task.getQueries().get(taskIndex);
|
263
|
-
if (task.getHttpMethod() == HttpMethod.GET) {
|
264
|
-
HttpGet request = new HttpGet(task.getUrl());
|
265
|
-
if (queries != null) {
|
266
|
-
URIBuilder builder = new URIBuilder(request.getURI());
|
267
|
-
for (QueryOption.Query q : queries) {
|
268
|
-
for (String v : q.getValues()) {
|
269
|
-
builder.addParameter(q.getName(), v);
|
270
|
-
}
|
271
|
-
}
|
272
|
-
request.setURI(builder.build());
|
273
|
-
}
|
274
|
-
return request;
|
275
|
-
}
|
276
|
-
if (task.getHttpMethod() == HttpMethod.POST) {
|
277
|
-
HttpPost request = new HttpPost(task.getUrl());
|
278
|
-
if (queries != null) {
|
279
|
-
List<NameValuePair> pairs = new ArrayList<>();
|
280
|
-
for (QueryOption.Query q : queries) {
|
281
|
-
for (String v : q.getValues()) {
|
282
|
-
pairs.add(new BasicNameValuePair(q.getName(), v));
|
283
|
-
}
|
284
|
-
}
|
285
|
-
request.setEntity(new UrlEncodedFormEntity(pairs));
|
286
|
-
}
|
287
|
-
else if (task.getRequestBody().isPresent()) {
|
288
|
-
logger.info(new StringEntity(task.getRequestBody().get()).toString());
|
289
|
-
request.setEntity(new StringEntity(task.getRequestBody().get()));
|
290
|
-
}
|
291
|
-
return request;
|
292
|
-
}
|
293
|
-
throw new IllegalArgumentException(String.format("Unsupported http method %s", task.getMethod()));
|
319
|
+
public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
|
320
|
+
super(task.getBufferAllocator(), new SingleFileProvider(stream));
|
321
|
+
this.startTimeMills = startTimeMills;
|
322
|
+
this.task = task;
|
294
323
|
}
|
295
324
|
|
296
|
-
|
297
|
-
|
298
|
-
List<Header> headers = new ArrayList<>();
|
299
|
-
headers.add(new BasicHeader("Accept", "*/*"));
|
300
|
-
headers.add(new BasicHeader("Accept-Charset", task.getCharset()));
|
301
|
-
headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
|
302
|
-
headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
|
303
|
-
headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
|
304
|
-
for (Map.Entry<String, String> entry : task.getRequestHeaders().entrySet()) {
|
305
|
-
headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
|
306
|
-
}
|
307
|
-
return headers;
|
325
|
+
public TaskReport commit() {
|
326
|
+
return Exec.newTaskReport();
|
308
327
|
}
|
309
328
|
|
310
|
-
|
311
|
-
{
|
312
|
-
|
313
|
-
|
314
|
-
.setMaxRedirects(10)
|
315
|
-
.setRedirectsEnabled(true)
|
316
|
-
.setConnectTimeout(task.getOpenTimeout())
|
317
|
-
.setSocketTimeout(task.getReadTimeout())
|
318
|
-
.build();
|
329
|
+
@Override
|
330
|
+
public void close() {
|
331
|
+
super.close();
|
332
|
+
handleInterval();
|
319
333
|
}
|
320
334
|
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
return Exec.newTaskReport();
|
335
|
+
@Override
|
336
|
+
public void abort() {}
|
337
|
+
|
338
|
+
protected void handleInterval() {
|
339
|
+
if (task.getRequestInterval() <= 0) {
|
340
|
+
return;
|
341
|
+
}
|
342
|
+
long interval = task.getRequestInterval();
|
343
|
+
if (task.getIntervalIncludesResponseTime()) {
|
344
|
+
interval = interval - (System.currentTimeMillis() - startTimeMills);
|
345
|
+
}
|
346
|
+
if (interval > 0) {
|
347
|
+
logger.info(String.format("waiting %d msec ...", interval));
|
348
|
+
try {
|
349
|
+
Thread.sleep(interval);
|
350
|
+
} catch (InterruptedException e) {
|
351
|
+
throw Throwables.propagate(e);
|
339
352
|
}
|
353
|
+
}
|
354
|
+
}
|
340
355
|
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
super.close();
|
345
|
-
handleInterval();
|
346
|
-
}
|
356
|
+
private static class SingleFileProvider implements InputStreamFileInput.Provider {
|
357
|
+
private final InputStream stream;
|
358
|
+
private boolean opened = false;
|
347
359
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
}
|
360
|
+
public SingleFileProvider(InputStream stream) {
|
361
|
+
this.stream = stream;
|
362
|
+
}
|
352
363
|
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
}
|
358
|
-
long interval = task.getRequestInterval();
|
359
|
-
if (task.getIntervalIncludesResponseTime()) {
|
360
|
-
interval = interval - (System.currentTimeMillis() - startTimeMills);
|
361
|
-
}
|
362
|
-
if (interval > 0) {
|
363
|
-
logger.info(String.format("waiting %d msec ...", interval));
|
364
|
-
try {
|
365
|
-
Thread.sleep(interval);
|
366
|
-
}
|
367
|
-
catch (InterruptedException e) {
|
368
|
-
throw Throwables.propagate(e);
|
369
|
-
}
|
370
|
-
}
|
364
|
+
@Override
|
365
|
+
public InputStream openNext() throws IOException {
|
366
|
+
if (opened) {
|
367
|
+
return null;
|
371
368
|
}
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
{
|
381
|
-
this.stream = stream;
|
382
|
-
}
|
383
|
-
|
384
|
-
@Override
|
385
|
-
public InputStream openNext() throws IOException
|
386
|
-
{
|
387
|
-
if (opened) {
|
388
|
-
return null;
|
389
|
-
}
|
390
|
-
opened = true;
|
391
|
-
return stream;
|
392
|
-
}
|
393
|
-
|
394
|
-
@Override
|
395
|
-
public void close() throws IOException
|
396
|
-
{
|
397
|
-
if (!opened) {
|
398
|
-
stream.close();
|
399
|
-
}
|
400
|
-
}
|
369
|
+
opened = true;
|
370
|
+
return stream;
|
371
|
+
}
|
372
|
+
|
373
|
+
@Override
|
374
|
+
public void close() throws IOException {
|
375
|
+
if (!opened) {
|
376
|
+
stream.close();
|
401
377
|
}
|
378
|
+
}
|
402
379
|
}
|
380
|
+
}
|
403
381
|
}
|