embulk-input-http 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -2
- data/build.gradle +3 -3
- data/example/json-example.yml +3 -11
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/input/HttpInputPlugin.java +62 -50
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4f5abf30041b0f9bf9e33f62351f722e04a8c5f9
|
4
|
+
data.tar.gz: 578bf2856f30946d8a116b09b52ff3ec7eadeaa4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 08d6de2aa365f241a5440765eb392523cb6cfa20b2aa75f34bcee6f698821a6921d478d5ec0f1b0ff8e24be84458a94ae7d28958d314aafdadd0dcfd7cd4ed17
|
7
|
+
data.tar.gz: e6996e01416a37b29a1fbab32f5f9c3334197efe646d073a905f924c48d11a7086e524bba1418f8897d60ea7d2b1110d73785d198d988890125ca8aca8b529fe
|
data/README.md
CHANGED
@@ -27,7 +27,7 @@ in:
|
|
27
27
|
method: get
|
28
28
|
```
|
29
29
|
|
30
|
-
- **type**: specify this plugin as
|
30
|
+
- **type**: specify this plugin as *http*
|
31
31
|
- **url**: base url something like api (required)
|
32
32
|
- **params**: pair of name/value to specify query parameter (optional)
|
33
33
|
- **method**: http method, get is used by default (optional)
|
@@ -38,7 +38,8 @@ in:
|
|
38
38
|
- **read_timeout**: timeout msec to read content via http (optional, 10000 is used by default)
|
39
39
|
- **max_retries**: max number of retry request if failed (optional, 5 is used by default)
|
40
40
|
- **retry_interval**: interval msec to retry max (optional, 10000 is used by default)
|
41
|
-
- **
|
41
|
+
- **request_interval**: wait msec before each requests (optional, 0 is used by default)
|
42
|
+
- **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, no is used by default)
|
42
43
|
|
43
44
|
### Brace expansion style in params
|
44
45
|
|
data/build.gradle
CHANGED
@@ -12,12 +12,12 @@ configurations {
|
|
12
12
|
provided
|
13
13
|
}
|
14
14
|
|
15
|
-
version = "0.0.
|
15
|
+
version = "0.0.9"
|
16
16
|
|
17
17
|
dependencies {
|
18
|
-
compile "org.embulk:embulk-core:0.7.
|
18
|
+
compile "org.embulk:embulk-core:0.7.4"
|
19
19
|
compile "org.apache.httpcomponents:httpclient:4.4"
|
20
|
-
provided "org.embulk:embulk-core:0.7.
|
20
|
+
provided "org.embulk:embulk-core:0.7.4"
|
21
21
|
testCompile "junit:junit:4.+"
|
22
22
|
}
|
23
23
|
|
data/example/json-example.yml
CHANGED
@@ -11,18 +11,10 @@ in:
|
|
11
11
|
method: get
|
12
12
|
user_agent: example_json
|
13
13
|
charset: utf8
|
14
|
+
interval_includes_response_time: yes
|
15
|
+
request_interval: 600
|
14
16
|
parser:
|
15
|
-
type:
|
16
|
-
root: $.response.station
|
17
|
-
schema:
|
18
|
-
- {name: name, type: string}
|
19
|
-
- {name: next, type: string}
|
20
|
-
- {name: prev, type: string}
|
21
|
-
- {name: distance, type: string}
|
22
|
-
- {name: lat, type: double, path: x}
|
23
|
-
- {name: lng, type: double, path: y}
|
24
|
-
- {name: line, type: string}
|
25
|
-
- {name: postal, type: string}
|
17
|
+
type: none
|
26
18
|
|
27
19
|
out: {type: stdout}
|
28
20
|
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Fri Jan 01 14:42:15 JST 2016
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-all.zip
|
@@ -71,11 +71,15 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
71
71
|
@ConfigDefault("10000")
|
72
72
|
public int getRetryInterval();
|
73
73
|
|
74
|
-
@Config("
|
74
|
+
@Config("request_interval")
|
75
75
|
@ConfigDefault("0")
|
76
|
-
public int
|
76
|
+
public int getRequestInterval();
|
77
77
|
|
78
|
-
public void
|
78
|
+
public void setRequestInterval(int requestInterval);
|
79
|
+
|
80
|
+
@Config("interval_includes_response_time")
|
81
|
+
@ConfigDefault("null")
|
82
|
+
public boolean getIntervalIncludesResponseTime();
|
79
83
|
|
80
84
|
@Config("params")
|
81
85
|
@ConfigDefault("null")
|
@@ -106,31 +110,20 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
106
110
|
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
|
107
111
|
PluginTask task = config.loadConfig(PluginTask.class);
|
108
112
|
|
109
|
-
int
|
113
|
+
final int tasks;
|
110
114
|
if (task.getParams().isPresent()) {
|
111
115
|
List<List<QueryConfig.Query>> expandedQueries = task.getParams().get().expandQueries();
|
112
116
|
task.setQueries(expandedQueries);
|
113
|
-
|
117
|
+
tasks = expandedQueries.size();
|
114
118
|
} else {
|
115
119
|
task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
|
120
|
+
task.setRequestInterval(0);
|
121
|
+
tasks = 1;
|
116
122
|
}
|
117
123
|
|
118
|
-
|
119
|
-
task.setSleepBeforeRequest(0);
|
120
|
-
}
|
121
|
-
|
122
|
-
switch (task.getMethod().toUpperCase()) {
|
123
|
-
case "GET":
|
124
|
-
task.setHttpMethod(HttpMethod.GET);
|
125
|
-
break;
|
126
|
-
case "POST":
|
127
|
-
task.setHttpMethod(HttpMethod.POST);
|
128
|
-
break;
|
129
|
-
default:
|
130
|
-
throw new ConfigException(String.format("Unsupported http method %s", task.getMethod()));
|
131
|
-
}
|
124
|
+
task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
|
132
125
|
|
133
|
-
return resume(task.dump(),
|
126
|
+
return resume(task.dump(), tasks, control);
|
134
127
|
}
|
135
128
|
|
136
129
|
@Override
|
@@ -142,9 +135,7 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
142
135
|
}
|
143
136
|
|
144
137
|
@Override
|
145
|
-
public void cleanup(TaskSource taskSource,
|
146
|
-
int taskCount,
|
147
|
-
List<TaskReport> successTaskReports) {
|
138
|
+
public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
|
148
139
|
}
|
149
140
|
|
150
141
|
@Override
|
@@ -164,34 +155,28 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
164
155
|
.setDefaultHeaders(makeHeaders(task));
|
165
156
|
|
166
157
|
if (task.getBasicAuth().isPresent()) {
|
167
|
-
builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(),
|
168
|
-
request));
|
158
|
+
builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(), request));
|
169
159
|
}
|
170
160
|
|
171
161
|
HttpClient client = builder.build();
|
172
162
|
|
173
|
-
logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
|
174
|
-
request.getURI().toString()));
|
163
|
+
logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(), request.getURI().toString()));
|
175
164
|
|
176
165
|
RetryableHandler retryable = new RetryableHandler(client, request);
|
166
|
+
long startTimeMills = System.currentTimeMillis();
|
177
167
|
try {
|
178
|
-
if (task.getSleepBeforeRequest() > 0) {
|
179
|
-
logger.info(String.format("wait %d msec before request", task.getSleepBeforeRequest()));
|
180
|
-
Thread.sleep(task.getSleepBeforeRequest());
|
181
|
-
}
|
182
168
|
RetryExecutor.retryExecutor().
|
183
169
|
withRetryLimit(task.getMaxRetries()).
|
184
170
|
withInitialRetryWait(task.getRetryInterval()).
|
185
171
|
withMaxRetryWait(30 * 60 * 1000).
|
186
172
|
runInterruptible(retryable);
|
187
173
|
InputStream stream = retryable.getResponse().getEntity().getContent();
|
188
|
-
PluginFileInput input = new PluginFileInput(task, stream);
|
174
|
+
PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
|
189
175
|
stream = null;
|
190
176
|
return input;
|
191
177
|
} catch (Exception e) {
|
192
178
|
throw Throwables.propagate(e);
|
193
179
|
}
|
194
|
-
|
195
180
|
}
|
196
181
|
|
197
182
|
private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
|
@@ -258,10 +243,53 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
258
243
|
public static class PluginFileInput extends InputStreamFileInput
|
259
244
|
implements TransactionalFileInput {
|
260
245
|
|
246
|
+
private final Logger logger = Exec.getLogger(getClass());
|
247
|
+
|
248
|
+
private final long startTimeMills;
|
249
|
+
private final PluginTask task;
|
250
|
+
|
251
|
+
public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
|
252
|
+
super(task.getBufferAllocator(), new SingleFileProvider(stream));
|
253
|
+
this.startTimeMills = startTimeMills;
|
254
|
+
this.task = task;
|
255
|
+
}
|
256
|
+
|
257
|
+
public TaskReport commit() {
|
258
|
+
return Exec.newTaskReport();
|
259
|
+
}
|
260
|
+
|
261
|
+
@Override
|
262
|
+
public void close() {
|
263
|
+
super.close();
|
264
|
+
handleInterval();
|
265
|
+
}
|
266
|
+
|
267
|
+
@Override
|
268
|
+
public void abort() {
|
269
|
+
}
|
270
|
+
|
271
|
+
protected void handleInterval() {
|
272
|
+
if (task.getRequestInterval() <= 0) {
|
273
|
+
return;
|
274
|
+
}
|
275
|
+
long interval = task.getRequestInterval();
|
276
|
+
if (task.getIntervalIncludesResponseTime()) {
|
277
|
+
interval = interval - (System.currentTimeMillis() - startTimeMills);
|
278
|
+
}
|
279
|
+
if (interval > 0) {
|
280
|
+
logger.info(String.format("waiting %d msec ...", interval));
|
281
|
+
try {
|
282
|
+
Thread.sleep(interval);
|
283
|
+
} catch (InterruptedException e) {
|
284
|
+
Throwables.propagate(e);
|
285
|
+
}
|
286
|
+
}
|
287
|
+
}
|
288
|
+
|
261
289
|
private static class SingleFileProvider
|
262
290
|
implements InputStreamFileInput.Provider {
|
263
291
|
|
264
|
-
private InputStream stream;
|
292
|
+
private final InputStream stream;
|
265
293
|
private boolean opened = false;
|
266
294
|
|
267
295
|
public SingleFileProvider(InputStream stream) {
|
@@ -284,21 +312,5 @@ public class HttpInputPlugin implements FileInputPlugin {
|
|
284
312
|
}
|
285
313
|
}
|
286
314
|
}
|
287
|
-
|
288
|
-
public PluginFileInput(PluginTask task, InputStream stream) {
|
289
|
-
super(task.getBufferAllocator(), new SingleFileProvider(stream));
|
290
|
-
}
|
291
|
-
|
292
|
-
public void abort() {
|
293
|
-
}
|
294
|
-
|
295
|
-
public TaskReport commit() {
|
296
|
-
return Exec.newTaskReport();
|
297
|
-
}
|
298
|
-
|
299
|
-
@Override
|
300
|
-
public void close() {
|
301
|
-
}
|
302
315
|
}
|
303
|
-
|
304
316
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-http
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Takuma kanari
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-01-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -65,7 +65,7 @@ files:
|
|
65
65
|
- src/test/java/org/embulk/input/TestQueryConfig.java
|
66
66
|
- classpath/commons-codec-1.9.jar
|
67
67
|
- classpath/commons-logging-1.2.jar
|
68
|
-
- classpath/embulk-input-http-0.0.
|
68
|
+
- classpath/embulk-input-http-0.0.9.jar
|
69
69
|
- classpath/httpclient-4.4.jar
|
70
70
|
- classpath/httpcore-4.4.jar
|
71
71
|
homepage: https://github.com/takumakanari/embulk-input-http
|