embulk-input-http 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35a826fa02c2999c31850703296d646f9e2067e7
-  data.tar.gz: 7240909008fb44cdcc5894e97c65b20474e42bd0
+  metadata.gz: 4f5abf30041b0f9bf9e33f62351f722e04a8c5f9
+  data.tar.gz: 578bf2856f30946d8a116b09b52ff3ec7eadeaa4
 SHA512:
-  metadata.gz: 58005db1c171527da4c7a35377ccd9983c17b4def76c77843d4f4e5061fb69d3c6f0cfbe5f6f41300a8952f4c3126fa63cec765f008342e50043912f9af5314d
-  data.tar.gz: d067a2ef1b224dac1fd037d24eb61e6fca6014ee475db211f4c5f65ccd88987c47ddf7585b257dbc57e3d492778772b80634612a1900e2c79ced1dd4a42149a6
+  metadata.gz: 08d6de2aa365f241a5440765eb392523cb6cfa20b2aa75f34bcee6f698821a6921d478d5ec0f1b0ff8e24be84458a94ae7d28958d314aafdadd0dcfd7cd4ed17
+  data.tar.gz: e6996e01416a37b29a1fbab32f5f9c3334197efe646d073a905f924c48d11a7086e524bba1418f8897d60ea7d2b1110d73785d198d988890125ca8aca8b529fe

data/README.md

@@ -27,7 +27,7 @@ in:
   method: get
 ```
 
-- **type**: specify this plugin as `http`
+- **type**: specify this plugin as *http*
 - **url**: base url something like api (required)
 - **params**: pair of name/value to specify query parameter (optional)
 - **method**: http method, get is used by default (optional)
@@ -38,7 +38,8 @@ in:
 - **read_timeout**: timeout msec to read content via http (optional, 10000 is used by default)
 - **max_retries**: max number of retry request if failed (optional, 5 is used by default)
 - **retry_interval**: interval msec to retry max (optional, 10000 is used by default)
-- **sleep\_before\_request**: wait msec before each requests (optional, 0 is used by default)
+- **request_interval**: wait msec before each requests (optional, 0 is used by default)
+- **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, no is used by default)
 
 ### Brace expansion style in params
 

data/build.gradle

@@ -12,12 +12,12 @@ configurations {
     provided
 }
 
-version = "0.0.8"
+version = "0.0.9"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.0"
+    compile  "org.embulk:embulk-core:0.7.4"
     compile  "org.apache.httpcomponents:httpclient:4.4"
-    provided "org.embulk:embulk-core:0.7.0"
+    provided "org.embulk:embulk-core:0.7.4"
     testCompile "junit:junit:4.+"
 }
 

data/example/json-example.yml

@@ -11,18 +11,10 @@ in:
   method: get
   user_agent: example_json
   charset: utf8
+  interval_includes_response_time: yes
+  request_interval: 600
   parser:
-    type: json
-    root: $.response.station
-    schema:
-      - {name: name, type: string}
-      - {name: next, type: string}
-      - {name: prev, type: string}
-      - {name: distance, type: string}
-      - {name: lat, type: double, path: x}
-      - {name: lng, type: double, path: y}
-      - {name: line, type: string}
-      - {name: postal, type: string}
+    type: none
 
 out: {type: stdout}
 

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Fri Jan 01 14:42:15 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-all.zip

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -71,11 +71,15 @@ public class HttpInputPlugin implements FileInputPlugin {
         @ConfigDefault("10000")
         public int getRetryInterval();
 
-        @Config("sleep_before_request")
+        @Config("request_interval")
         @ConfigDefault("0")
-        public int getSleepBeforeRequest();
+        public int getRequestInterval();
 
-        public void setSleepBeforeRequest(int sleepBeforeRequest);
+        public void setRequestInterval(int requestInterval);
+
+        @Config("interval_includes_response_time")
+        @ConfigDefault("null")
+        public boolean getIntervalIncludesResponseTime();
 
         @Config("params")
         @ConfigDefault("null")
@@ -106,31 +110,20 @@ public class HttpInputPlugin implements FileInputPlugin {
     public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        int numOfThreads = 1;
+        final int tasks;
         if (task.getParams().isPresent()) {
             List<List<QueryConfig.Query>> expandedQueries = task.getParams().get().expandQueries();
             task.setQueries(expandedQueries);
-            numOfThreads = expandedQueries.size();
+            tasks = expandedQueries.size();
         } else {
             task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
+            task.setRequestInterval(0);
+            tasks = 1;
         }
 
-        if (numOfThreads == 1) {
-            task.setSleepBeforeRequest(0);
-        }
-
-        switch (task.getMethod().toUpperCase()) {
-            case "GET":
-                task.setHttpMethod(HttpMethod.GET);
-                break;
-            case "POST":
-                task.setHttpMethod(HttpMethod.POST);
-                break;
-            default:
-                throw new ConfigException(String.format("Unsupported http method %s", task.getMethod()));
-        }
+        task.setHttpMethod(HttpMethod.valueOf(task.getMethod().toUpperCase()));
 
-        return resume(task.dump(), numOfThreads, control);
+        return resume(task.dump(), tasks, control);
     }
 
     @Override
@@ -142,9 +135,7 @@ public class HttpInputPlugin implements FileInputPlugin {
     }
 
     @Override
-    public void cleanup(TaskSource taskSource,
-                        int taskCount,
-                        List<TaskReport> successTaskReports) {
+    public void cleanup(TaskSource taskSource, int taskCount, List<TaskReport> successTaskReports) {
     }
 
     @Override
@@ -164,34 +155,28 @@ public class HttpInputPlugin implements FileInputPlugin {
                 .setDefaultHeaders(makeHeaders(task));
 
         if (task.getBasicAuth().isPresent()) {
-            builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(),
-                    request));
+            builder.setDefaultCredentialsProvider(makeCredentialsProvider(task.getBasicAuth().get(), request));
         }
 
         HttpClient client = builder.build();
 
-        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(),
-                request.getURI().toString()));
+        logger.info(String.format("%s \"%s\"", task.getMethod().toUpperCase(), request.getURI().toString()));
 
         RetryableHandler retryable = new RetryableHandler(client, request);
+        long startTimeMills = System.currentTimeMillis();
         try {
-            if (task.getSleepBeforeRequest() > 0) {
-                logger.info(String.format("wait %d msec before request", task.getSleepBeforeRequest()));
-                Thread.sleep(task.getSleepBeforeRequest());
-            }
             RetryExecutor.retryExecutor().
                     withRetryLimit(task.getMaxRetries()).
                     withInitialRetryWait(task.getRetryInterval()).
                     withMaxRetryWait(30 * 60 * 1000).
                     runInterruptible(retryable);
             InputStream stream = retryable.getResponse().getEntity().getContent();
-            PluginFileInput input = new PluginFileInput(task, stream);
+            PluginFileInput input = new PluginFileInput(task, stream, startTimeMills);
             stream = null;
             return input;
         } catch (Exception e) {
             throw Throwables.propagate(e);
         }
-
     }
 
     private CredentialsProvider makeCredentialsProvider(BasicAuthConfig config, HttpRequestBase scopeRequest) {
@@ -258,10 +243,53 @@ public class HttpInputPlugin implements FileInputPlugin {
     public static class PluginFileInput extends InputStreamFileInput
             implements TransactionalFileInput {
 
+        private final Logger logger = Exec.getLogger(getClass());
+
+        private final long startTimeMills;
+        private final PluginTask task;
+
+        public PluginFileInput(PluginTask task, InputStream stream, long startTimeMills) {
+            super(task.getBufferAllocator(), new SingleFileProvider(stream));
+            this.startTimeMills = startTimeMills;
+            this.task = task;
+        }
+
+        public TaskReport commit() {
+            return Exec.newTaskReport();
+        }
+
+        @Override
+        public void close() {
+            super.close();
+            handleInterval();
+        }
+
+        @Override
+        public void abort() {
+        }
+
+        protected void handleInterval() {
+            if (task.getRequestInterval() <= 0) {
+                return;
+            }
+            long interval = task.getRequestInterval();
+            if (task.getIntervalIncludesResponseTime()) {
+                interval = interval - (System.currentTimeMillis() - startTimeMills);
+            }
+            if (interval > 0) {
+                logger.info(String.format("waiting %d msec ...", interval));
+                try {
+                    Thread.sleep(interval);
+                } catch (InterruptedException e) {
+                    Throwables.propagate(e);
+                }
+            }
+        }
+
         private static class SingleFileProvider
                 implements InputStreamFileInput.Provider {
 
-            private InputStream stream;
+            private final InputStream stream;
             private boolean opened = false;
 
             public SingleFileProvider(InputStream stream) {
@@ -284,21 +312,5 @@ public class HttpInputPlugin implements FileInputPlugin {
                 }
             }
         }
-
-        public PluginFileInput(PluginTask task, InputStream stream) {
-            super(task.getBufferAllocator(), new SingleFileProvider(stream));
-        }
-
-        public void abort() {
-        }
-
-        public TaskReport commit() {
-            return Exec.newTaskReport();
-        }
-
-        @Override
-        public void close() {
-        }
     }
-
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-http
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
 platform: ruby
 authors:
 - Takuma kanari
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2016-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/input/TestQueryConfig.java
 - classpath/commons-codec-1.9.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-http-0.0.8.jar
+- classpath/embulk-input-http-0.0.9.jar
 - classpath/httpclient-4.4.jar
 - classpath/httpcore-4.4.jar
 homepage: https://github.com/takumakanari/embulk-input-http