embulk-input-http 0.0.8 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 35a826fa02c2999c31850703296d646f9e2067e7
4
- data.tar.gz: 7240909008fb44cdcc5894e97c65b20474e42bd0
3
+ metadata.gz: 4f5abf30041b0f9bf9e33f62351f722e04a8c5f9
4
+ data.tar.gz: 578bf2856f30946d8a116b09b52ff3ec7eadeaa4
5
5
  SHA512:
6
- metadata.gz: 58005db1c171527da4c7a35377ccd9983c17b4def76c77843d4f4e5061fb69d3c6f0cfbe5f6f41300a8952f4c3126fa63cec765f008342e50043912f9af5314d
7
- data.tar.gz: d067a2ef1b224dac1fd037d24eb61e6fca6014ee475db211f4c5f65ccd88987c47ddf7585b257dbc57e3d492778772b80634612a1900e2c79ced1dd4a42149a6
6
+ metadata.gz: 08d6de2aa365f241a5440765eb392523cb6cfa20b2aa75f34bcee6f698821a6921d478d5ec0f1b0ff8e24be84458a94ae7d28958d314aafdadd0dcfd7cd4ed17
7
+ data.tar.gz: e6996e01416a37b29a1fbab32f5f9c3334197efe646d073a905f924c48d11a7086e524bba1418f8897d60ea7d2b1110d73785d198d988890125ca8aca8b529fe
data/README.md CHANGED
@@ -27,7 +27,7 @@ in:
27
27
  method: get
28
28
  ```
29
29
 
30
- - **type**: specify this plugin as `http`
30
+ - **type**: specify this plugin as *http*
31
31
  - **url**: base url something like api (required)
32
32
  - **params**: pair of name/value to specify query parameter (optional)
33
33
  - **method**: http method, get is used by default (optional)
@@ -38,7 +38,8 @@ in:
38
38
  - **read_timeout**: timeout msec to read content via http (optional, 10000 is used by default)
39
39
  - **max_retries**: max number of retry request if failed (optional, 5 is used by default)
40
40
  - **retry_interval**: interval msec to retry max (optional, 10000 is used by default)
41
- - **sleep\_before\_request**: wait msec before each requests (optional, 0 is used by default)
41
+ - **request_interval**: wait msec before each requests (optional, 0 is used by default)
42
+ - **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, no is used by default)
42
43
 
43
44
  ### Brace expansion style in params
44
45
 
data/build.gradle CHANGED
@@ -12,12 +12,12 @@ configurations {
12
12
  provided
13
13
  }
14
14
 
15
- version = "0.0.8"
15
+ version = "0.0.9"
16
16
 
17
17
  dependencies {
18
- compile "org.embulk:embulk-core:0.7.0"
18
+ compile "org.embulk:embulk-core:0.7.4"
19
19
  compile "org.apache.httpcomponents:httpclient:4.4"
20
- provided "org.embulk:embulk-core:0.7.0"
20
+ provided "org.embulk:embulk-core:0.7.4"
21
21
  testCompile "junit:junit:4.+"
22
22
  }
23
23
 
@@ -11,18 +11,10 @@ in:
11
11
  method: get
12
12
  user_agent: example_json
13
13
  charset: utf8
14
+ interval_includes_response_time: yes
15
+ request_interval: 600
14
16
  parser:
15
- type: json
16
- root: $.response.station
17
- schema:
18
- - {name: name, type: string}
19
- - {name: next, type: string}
20
- - {name: prev, type: string}
21
- - {name: distance, type: string}
22
- - {name: lat, type: double, path: x}
23
- - {name: lng, type: double, path: y}
24
- - {name: line, type: string}
25
- - {name: postal, type: string}
17
+ type: none
26
18
 
27
19
  out: {type: stdout}
28
20
 
@@ -1,6 +1,6 @@
1
- #Tue Aug 11 00:26:20 PDT 2015
1
+ #Fri Jan 01 14:42:15 JST 2016
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-all.zip
@@ -71,11 +71,15 @@ public class HttpInputPlugin implements FileInputPlugin {
71
71
  @ConfigDefault("10000")
72
72
  public int getRetryInterval();
73
73
 
74
- @Config("sleep_before_request")
74
+ @Config("request_interval")
75
75
  @ConfigDefault("0")
76
- public int getSleepBeforeRequest();
76
+ public int getRequestInterval();
77
77
 
78
- public void setSleepBeforeRequest(int sleepBeforeRequest);
78
+ public void setRequestInterval(int requestInterval);
79
+
80
+ @Config("interval_includes_response_time")
81
+ @ConfigDefault("null")
82
+ public boolean getIntervalIncludesResponseTime();
79
83
 
80
84
  @Config("params")
81
85
  @ConfigDefault("null")
@@ -106,31 +110,20 @@ public class HttpInputPlugin implements FileInputPlugin {
106
110
  public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
107
111
  PluginTask task = config.loadConfig(PluginTask.class);
108
112
 
109
- int numOfThreads = 1;
113
+ final int tasks;
110
114
  if (task.getParams().isPresent()) {
111
115
  List<List<QueryConfig.Query>> expandedQueries = task.getParams().get().expandQueries();
112
116
  task.setQueries(expandedQueries);
113
- numOfThreads = expandedQueries.size();
117
+ tasks = expandedQueries.size();
114
118
  } else {
115
119
  task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
120
+ task.setRequestInterval(0);
121
+ tasks = 1;
116
122
  }
117
123
 
118
- if (numOfThreads == 1) {
119
- task.setSleepBeforeRequest(0);
120
- }
121
-
122
- switch (task.getMethod().toUpperCase()) {
123
- case "GET":
124
- task.setHttpMethod(HttpMethod.GET);
125
- break;
126
- case "POST":
127
- task.setHttpMethod(HttpMethod.POST);
128
- break;
129
- default:
130
- throw new ConfigException(String.format("Unsupported http method %s", task.getMethod()));
131
- }
124
+ task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
132
125
 
133
- return resume(task.dump(), numOfThreads, control);
126
+ return resume(task.dump(), tasks, control);
134
127
  }
135
128
 
136
129
  @Override
@@ -142,9 +135,7 @@ public class HttpInputPlugin implements FileInputPlugin {
142
135
  }
143
136
 
144
137
  @Override
145
- public void cleanup(TaskSource taskSource,
146
- int taskCount,
147
- List<TaskReport> successTaskReports) {
138
+ public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
148
139
  }
149
140
 
150
141
  @Override
@@ -164,34 +155,28 @@ public class HttpInputPlugin implements FileInputPlugin {
164
155
  .setDefaultHeaders(makeHeaders(task));
165
156
 
166
157
  if (task.getBasicAuth().isPresent()) {
167
- builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(),
168
- request));
158
+ builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(), request));
169
159
  }
170
160
 
171
161
  HttpClient client = builder.build();
172
162
 
173
- logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
174
- request.getURI().toString()));
163
+ logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(), request.getURI().toString()));
175
164
 
176
165
  RetryableHandler retryable = new RetryableHandler(client, request);
166
+ long startTimeMills = System.currentTimeMillis();
177
167
  try {
178
- if (task.getSleepBeforeRequest() > 0) {
179
- logger.info(String.format("wait %d msec before request", task.getSleepBeforeRequest()));
180
- Thread.sleep(task.getSleepBeforeRequest());
181
- }
182
168
  RetryExecutor.retryExecutor().
183
169
  withRetryLimit(task.getMaxRetries()).
184
170
  withInitialRetryWait(task.getRetryInterval()).
185
171
  withMaxRetryWait(30 * 60 * 1000).
186
172
  runInterruptible(retryable);
187
173
  InputStream stream = retryable.getResponse().getEntity().getContent();
188
- PluginFileInput input = new PluginFileInput(task, stream);
174
+ PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
189
175
  stream = null;
190
176
  return input;
191
177
  } catch (Exception e) {
192
178
  throw Throwables.propagate(e);
193
179
  }
194
-
195
180
  }
196
181
 
197
182
  private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
@@ -258,10 +243,53 @@ public class HttpInputPlugin implements FileInputPlugin {
258
243
  public static class PluginFileInput extends InputStreamFileInput
259
244
  implements TransactionalFileInput {
260
245
 
246
+ private final Logger logger = Exec.getLogger(getClass());
247
+
248
+ private final long startTimeMills;
249
+ private final PluginTask task;
250
+
251
+ public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
252
+ super(task.getBufferAllocator(), new SingleFileProvider(stream));
253
+ this.startTimeMills = startTimeMills;
254
+ this.task = task;
255
+ }
256
+
257
+ public TaskReport commit() {
258
+ return Exec.newTaskReport();
259
+ }
260
+
261
+ @Override
262
+ public void close() {
263
+ super.close();
264
+ handleInterval();
265
+ }
266
+
267
+ @Override
268
+ public void abort() {
269
+ }
270
+
271
+ protected void handleInterval() {
272
+ if (task.getRequestInterval() <= 0) {
273
+ return;
274
+ }
275
+ long interval = task.getRequestInterval();
276
+ if (task.getIntervalIncludesResponseTime()) {
277
+ interval = interval - (System.currentTimeMillis() - startTimeMills);
278
+ }
279
+ if (interval > 0) {
280
+ logger.info(String.format("waiting %d msec ...", interval));
281
+ try {
282
+ Thread.sleep(interval);
283
+ } catch (InterruptedException e) {
284
+ Throwables.propagate(e);
285
+ }
286
+ }
287
+ }
288
+
261
289
  private static class SingleFileProvider
262
290
  implements InputStreamFileInput.Provider {
263
291
 
264
- private InputStream stream;
292
+ private final InputStream stream;
265
293
  private boolean opened = false;
266
294
 
267
295
  public SingleFileProvider(InputStream stream) {
@@ -284,21 +312,5 @@ public class HttpInputPlugin implements FileInputPlugin {
284
312
  }
285
313
  }
286
314
  }
287
-
288
- public PluginFileInput(PluginTask task, InputStream stream) {
289
- super(task.getBufferAllocator(), new SingleFileProvider(stream));
290
- }
291
-
292
- public void abort() {
293
- }
294
-
295
- public TaskReport commit() {
296
- return Exec.newTaskReport();
297
- }
298
-
299
- @Override
300
- public void close() {
301
- }
302
315
  }
303
-
304
316
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-http
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.8
4
+ version: 0.0.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Takuma kanari
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-19 00:00:00.000000000 Z
11
+ date: 2016-01-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
65
65
  - src/test/java/org/embulk/input/TestQueryConfig.java
66
66
  - classpath/commons-codec-1.9.jar
67
67
  - classpath/commons-logging-1.2.jar
68
- - classpath/embulk-input-http-0.0.8.jar
68
+ - classpath/embulk-input-http-0.0.9.jar
69
69
  - classpath/httpclient-4.4.jar
70
70
  - classpath/httpcore-4.4.jar
71
71
  homepage: https://github.com/takumakanari/embulk-input-http