embulk-input-http 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 913d001d72650ad4679b8096fe04f54a937f7926
-  data.tar.gz: 9af3fe59316a75a451a96469f58412ae769c07e1
+  metadata.gz: 452759fbe56ecd2ad188fa9a242632e336cc0d36
+  data.tar.gz: 120395d8cfa06b72c150346d06b22f3b864e09a4
 SHA512:
-  metadata.gz: 948497dd6ec01d497b478730a4bb22a53a2c3ed1e35ce50ae9d24c3284e8655cd1f7ceab19555e7b4fb538ad80e0199590781a31866b4b0925fbec02ea5414d0
-  data.tar.gz: 3e5d676d85d6c070f29a33a9dbe7795569b6538e05cc3d8c79bf10299492631d006236fc48622c78d49ab0b8a4d203d2e8c90a4cc221033f45c1eced4856d5a0
+  metadata.gz: 10d8542672bd947506f551c35daf0e8e76638962ac96818e4feaa7603425cc84c858066f72ecd91b43e71f870ec1b01a1a497f301186a0ff52f1543c19184af1
+  data.tar.gz: 808141162ec7c55a29d6a207ccdaa33843c2bdf774c0ba4c3e9dffe81f8df9a198540717453f74007c79649db733560595c7c558f1349e719d5e93c23a4fc253

data/README.md

@@ -30,6 +30,7 @@ in:
 - **type**: specify this plugin as *http*
 - **url**: base url something like api (required)
 - **params**: pair of name/value to specify query parameter (optional)
+- **pager**: configuration to parameterize paging (optional)
 - **method**: http method, get is used by default (optional)
 - **user_agent**: the usrr agent to specify request header (optional)
 - **request_headers**: the extra request headers as key-value (optional)
@@ -42,7 +43,7 @@ in:
 - **request_interval**: wait msec before each requests (optional, 0 is used by default)
 - **interval\_includes\_response\_time**: yes/no, if yes and you set *request_interval*, response time will be included in interval for next request (optional, no is used by default)
 
-### Brace expansion style in params
+### Brace expansion style in 'params'
 
 In *params* section, you can specify also multilple requests by using **values** or **brace expansion style** with set **expand** true.
 
@@ -67,7 +68,6 @@ Then all patterns of query will be called in a defferent request.
 
 By default, **expand** is false. In this case, all values will be multiple params in one request.
 
-
 ### Use basic authentication
 
 You can specify username/password for basic authentication.
@@ -78,6 +78,56 @@ basic_auth:
  - password: MyPassword
 ```
 
+### Paging by 'pager'
+
+You can configure request parameters for paging requests, like as follows:
+
+```yaml
+in:
+  type: http
+  url: http://express.heartrails.com/api/json
+  pager: {from_param: from, to_param: to, start: 1, step: 1000, pages: 10}
+```
+
+Properties of pager is as below:
+
+- **from_param**: parameter name of 'from' index
+- **to_param**: parameter name of 'to' index (optional)
+- **pages**: total page size
+- **start**: first index number (optional, 0 is used by default)
+- **step**: size to increment (optional, 1 is used by default)
+
+#### Examples of 'pager'
+
+#### Conbination of from/to parameters.
+
+```yaml
+pager: {from_param: from, to_param: to, pages: 4, start: 1, step: 10}
+```
+
+the request parameters will be:
+
+1. ?from=1&to=10
+2. ?from=11&to=20
+3. ?from=21&to=30
+4. ?from=31&to=40
+
+
+##### Batch request with incremental page parameter.
+
+```yaml
+params:
+  - {name: size, value: 100}
+pager: {from_param: page, pages: 4, start: 1, step: 1}
+```
+
+the request parameters will be:
+
+1. ?page=1&size=100
+2. ?page=2&size=100
+3. ?page=3&size=100
+4. ?page=4&size=100
+
 
 ## Example
 
@@ -91,7 +141,7 @@ in:
     - {name: method, value: getStations}
     - {name: x, value: 135.0}
     - {name: y, value: "{35,34,33,32,31}.0", expand: true}
-　request_headers: {X-Some-Key1: some-value1, X-Some-key2: some-value2}
+  request_headers: {X-Some-Key1: some-value1, X-Some-key2: some-value2}
   parser:
     type: json
     root: $.response.station

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.12"
+version = "0.0.13"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/example/json-example.yml

@@ -6,14 +6,14 @@ in:
   params:
     - {name: method, value: getStations}
     - {name: x, value: 135.0}
-    - {name: y, value: "{35,34,33,32,31}.0", expand: 1}
-    - {name: z, values: ["a", "b", "c"], expand: 1}
+    - {name: y, value: "{35,34}.0", expand: 1}
   method: get
   user_agent: example_json
   charset: utf8
   request_interval: 600
   interval_includes_response_time: 1
   request_headers: {X-Some-Key1: some-value1, X-Some-key2: some-value2}
+  pager: {from_param: from, to_param: to, pages: 10, step: 100}
   parser:
     type: none
 

data/src/main/java/org/embulk/input/HttpInputPlugin.java

@@ -90,11 +90,14 @@ public class HttpInputPlugin implements FileInputPlugin {
         @ConfigDefault("null")
         public Optional<BasicAuthConfig> getBasicAuth();
 
+        @Config("pager")
+        @ConfigDefault("null")
+        public Optional<PagerConfig> getPager();
+
         @Config("request_headers")
         @ConfigDefault("{}")
         public Map<String, String> getRequestHeaders();
 
-
         @ConfigInject
         public BufferAllocator getBufferAllocator();
 
@@ -118,9 +121,13 @@ public class HttpInputPlugin implements FileInputPlugin {
 
         final int tasks;
         if (task.getParams().isPresent()) {
-            List<List<QueryConfig.Query>> expandedQueries = task.getParams().get().expandQueries();
-            task.setQueries(expandedQueries);
-            tasks = expandedQueries.size();
+            List<List<QueryConfig.Query>> queries = task.getParams().get().generateQueries(task.getPager());
+            task.setQueries(queries);
+            tasks = queries.size();
+        } else if (task.getPager().isPresent()) {
+            List<List<QueryConfig.Query>> queries = task.getPager().get().expand();
+            task.setQueries(queries);
+            tasks = queries.size();
         } else {
             task.setQueries(Lists.<List<QueryConfig.Query>>newArrayList());
             task.setRequestInterval(0);
@@ -134,8 +141,8 @@ public class HttpInputPlugin implements FileInputPlugin {
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-                             int taskCount,
-                             FileInputPlugin.Control control) {
+            int taskCount,
+            FileInputPlugin.Control control) {
         control.run(taskSource, taskCount);
         return Exec.newConfigDiff();
     }
@@ -233,7 +240,7 @@ public class HttpInputPlugin implements FileInputPlugin {
         headers.add(new BasicHeader("Accept-Encoding", "gzip, deflate"));
         headers.add(new BasicHeader("Accept-Language", "en-us,en;q=0.5"));
         headers.add(new BasicHeader("User-Agent", task.getUserAgent()));
-        for (Map.Entry<String,String> entry : task.getRequestHeaders().entrySet()) {
+        for (Map.Entry<String, String> entry : task.getRequestHeaders().entrySet()) {
             headers.add(new BasicHeader(entry.getKey(), entry.getValue()));
         }
         return headers;

data/src/main/java/org/embulk/input/PagerConfig.java

@@ -0,0 +1,86 @@
+package org.embulk.input;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PagerConfig {
+
+    private final String fromParam;
+    private final Optional<String> toParam;
+    private final int start;
+    private final int pages;
+    private final int step;
+
+    @JsonCreator
+    public PagerConfig(@JsonProperty("from_param") String fromParam,
+            @JsonProperty("to_param") Optional<String> toParam,
+            @JsonProperty("start") Optional<Integer> start,
+            @JsonProperty("pages") int pages,
+            @JsonProperty("step") Optional<Integer> step) {
+        this.fromParam = fromParam;
+        this.toParam = toParam;
+        this.start = start.or(0);
+        this.pages = pages;
+        this.step = step.or(1);
+    }
+
+    public List<List<QueryConfig.Query>> expand() {
+        List<List<QueryConfig.Query>> queries = new ArrayList<>();
+        int p = 1;
+        int index = start;
+        while (p <= pages) {
+            List<QueryConfig.Query> one = new ArrayList<>();
+            one.add(new QueryConfig.Query(fromParam, Integer.toString(index)));
+            if (toParam.isPresent()) {
+                int t = index + step - 1;
+                one.add(new QueryConfig.Query(toParam.get(), Integer.toString(t)));
+                index = t + 1;
+            } else {
+                index += step;
+            }
+            queries.add(one);
+            p++;
+        }
+        return queries;
+    }
+
+    @JsonProperty("from_param")
+    public String getFromParam() {
+        return fromParam;
+    }
+
+    @JsonProperty("to_param")
+    public Optional<String> getToParam() {
+        return toParam;
+    }
+
+    @JsonProperty("start")
+    public int getStart() {
+        return start;
+    }
+
+    @JsonProperty("pages")
+    public int getPages() {
+        return pages;
+    }
+
+    @JsonProperty("step")
+    public int getStep() {
+        return step;
+    }
+
+    @Override
+    public String toString() {
+        return "PagerConfig{" +
+                "fromParam='" + fromParam + '\'' +
+                ", toParam=" + toParam +
+                ", start=" + start +
+                ", pages=" + pages +
+                ", step=" + step +
+                '}';
+    }
+}

data/src/main/java/org/embulk/input/ParamsConfig.java

@@ -3,6 +3,7 @@ package org.embulk.input;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Objects;
+import com.google.common.base.Optional;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -21,7 +22,7 @@ public class ParamsConfig {
         return queries;
     }
 
-    public List<List<QueryConfig.Query>> expandQueries() {
+    public List<List<QueryConfig.Query>> generateQueries(Optional<PagerConfig> pagerConfig) {
         List<List<QueryConfig.Query>> base = new ArrayList<>(queries.size());
         for (QueryConfig p : queries) {
             base.add(p.expand());
@@ -29,18 +30,24 @@ public class ParamsConfig {
 
         int productSize = 1;
         int baseSize = base.size();
-        for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++) ;
+        for (int i = 0; i < baseSize; productSize *= base.get(i).size(), i++);
 
         List<List<QueryConfig.Query>> expands = new ArrayList<>(productSize);
         for (int i = 0; i < productSize; i++) {
             int j = 1;
-            List<QueryConfig.Query> query = new ArrayList<>();
+            List<QueryConfig.Query> one = new ArrayList<>();
             for (List<QueryConfig.Query> list : base) {
                 QueryConfig.Query pc = list.get((i / j) % list.size());
-                query.add(pc);
+                one.add(pc);
                 j *= list.size();
             }
-            expands.add(query);
+            if (pagerConfig.isPresent()) {
+                for (List<QueryConfig.Query> q : pagerConfig.get().expand()) {
+                    expands.add(copyAndConcat(one, q));
+                }
+            } else {
+                expands.add(one);
+            }
         }
 
         return expands;
@@ -63,4 +70,13 @@ public class ParamsConfig {
         return Objects.hashCode(queries);
     }
 
+    private List<QueryConfig.Query> copyAndConcat(List<QueryConfig.Query>... srcs) {
+        List<QueryConfig.Query> dest = new ArrayList<>();
+        for (List<QueryConfig.Query> src : srcs) {
+            for (QueryConfig.Query q : src) {
+                dest.add(q.copy());
+            }
+        }
+        return dest;
+    }
 }

data/src/main/java/org/embulk/input/QueryConfig.java

@@ -6,6 +6,7 @@ import com.google.common.base.Objects;
 import com.google.common.base.Optional;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 
 public class QueryConfig {
@@ -115,6 +116,10 @@ public class QueryConfig {
         public String[] getValues() {
             return values;
         }
+
+        public Query copy() {
+            return new Query(this.name, Arrays.copyOf(this.values, this.values.length));
+        }
     }
 
     private static class BraceExpansion {

data/src/test/java/org/embulk/input/TestPagerConfig.java

@@ -0,0 +1,78 @@
+package org.embulk.input;
+
+import com.google.common.base.Optional;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestPagerConfig {
+
+    @Test
+    public void testExpandFromTo() throws Exception {
+        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), Optional.of(1), 3,
+                Optional.of(2)).expand();
+        assertEquals(dest.size(), 3);
+
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "from");
+        assertEquals(dest.get(0).get(0).getValues()[0], "1");
+        assertEquals(dest.get(0).get(1).getName(), "to");
+        assertEquals(dest.get(0).get(1).getValues()[0], "2");
+
+        assertEquals(dest.get(1).size(), 2);
+        assertEquals(dest.get(1).get(0).getName(), "from");
+        assertEquals(dest.get(1).get(0).getValues()[0], "3");
+        assertEquals(dest.get(1).get(1).getName(), "to");
+        assertEquals(dest.get(1).get(1).getValues()[0], "4");
+
+        assertEquals(dest.get(2).size(), 2);
+        assertEquals(dest.get(2).get(0).getName(), "from");
+        assertEquals(dest.get(2).get(0).getValues()[0], "5");
+        assertEquals(dest.get(2).get(1).getName(), "to");
+        assertEquals(dest.get(2).get(1).getValues()[0], "6");
+    }
+
+    @Test
+    public void testExpandFromToWithDefault() throws Exception {
+        Optional<Integer> nullValue = Optional.absent();
+
+        List<List<QueryConfig.Query>> dest = new PagerConfig("from", Optional.of("to"), nullValue, 2, nullValue)
+                .expand();
+        assertEquals(dest.size(), 2);
+
+        assertEquals(dest.get(0).size(), 2);
+        assertEquals(dest.get(0).get(0).getName(), "from");
+        assertEquals(dest.get(0).get(0).getValues()[0], "0");
+        assertEquals(dest.get(0).get(1).getName(), "to");
+        assertEquals(dest.get(0).get(1).getValues()[0], "0");
+
+        assertEquals(dest.get(1).size(), 2);
+        assertEquals(dest.get(1).get(0).getName(), "from");
+        assertEquals(dest.get(1).get(0).getValues()[0], "1");
+        assertEquals(dest.get(1).get(1).getName(), "to");
+        assertEquals(dest.get(1).get(1).getValues()[0], "1");
+    }
+
+    @Test
+    public void testExpandPagenate() throws Exception {
+        Optional<String> nullValue = Optional.absent();
+        List<List<QueryConfig.Query>> dest = new PagerConfig("page", nullValue, Optional.of(1), 3,
+                Optional.of(1)).expand();
+        assertEquals(dest.size(), 3);
+
+        assertEquals(dest.get(0).size(), 1);
+        assertEquals(dest.get(0).get(0).getName(), "page");
+        assertEquals(dest.get(0).get(0).getValues()[0], "1");
+
+        assertEquals(dest.get(1).size(), 1);
+        assertEquals(dest.get(1).get(0).getName(), "page");
+        assertEquals(dest.get(1).get(0).getValues()[0], "2");
+
+        assertEquals(dest.get(2).size(), 1);
+        assertEquals(dest.get(2).get(0).getName(), "page");
+        assertEquals(dest.get(2).get(0).getValues()[0], "3");
+    }
+
+}

data/src/test/java/org/embulk/input/TestParamsConfig.java

@@ -16,7 +16,8 @@ public class TestParamsConfig {
         QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, false);
         QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, false);
         ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        List<List<QueryConfig.Query>> dest = paramsConfig.expandQueries();
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
         assertEquals(dest.size(), 1);
         assertEquals(dest.get(0).size(), 2);
         assertEquals(dest.get(0).get(0).getName(), "test1");
@@ -35,7 +36,8 @@ public class TestParamsConfig {
         QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), false);
 
         ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        List<List<QueryConfig.Query>> dest = paramsConfig.expandQueries();
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
         assertEquals(dest.size(), 1);
         assertEquals(dest.get(0).size(), 2);
         assertEquals(dest.get(0).get(0).getName(), "test1");
@@ -52,7 +54,8 @@ public class TestParamsConfig {
         QueryConfig q1 = new QueryConfig("test1", Optional.of("awasome1"), nullValues, true);
         QueryConfig q2 = new QueryConfig("test2", Optional.of("awasome2"), nullValues, true);
         ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        List<List<QueryConfig.Query>> dest = paramsConfig.expandQueries();
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
         assertEquals(dest.size(), 1);
         assertEquals(dest.get(0).size(), 2);
         assertEquals(dest.get(0).get(0).getName(), "test1");
@@ -71,7 +74,8 @@ public class TestParamsConfig {
         QueryConfig q2 = new QueryConfig("test2", nullValue, Optional.of(values2), true);
 
         ParamsConfig paramsConfig = new ParamsConfig(Lists.newArrayList(q1, q2));
-        List<List<QueryConfig.Query>> dest = paramsConfig.expandQueries();
+        Optional<PagerConfig> pagerConfig = Optional.absent();
+        List<List<QueryConfig.Query>> dest = paramsConfig.generateQueries(pagerConfig);
         assertEquals(dest.size(), 4);
 
         assertEquals(dest.get(0).size(), 2);

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-http
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Takuma kanari
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-05 00:00:00.000000000 Z
+date: 2017-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Fetch data via http
 email:
 - chemtrails.t@gmail.com
@@ -60,15 +60,17 @@ files:
 - settings.gradle
 - src/main/java/org/embulk/input/BasicAuthConfig.java
 - src/main/java/org/embulk/input/HttpInputPlugin.java
+- src/main/java/org/embulk/input/PagerConfig.java
 - src/main/java/org/embulk/input/ParamsConfig.java
 - src/main/java/org/embulk/input/QueryConfig.java
 - src/main/java/org/embulk/input/RetryableHandler.java
 - src/test/java/org/embulk/input/TestHttpInputPlugin.java
+- src/test/java/org/embulk/input/TestPagerConfig.java
 - src/test/java/org/embulk/input/TestParamsConfig.java
 - src/test/java/org/embulk/input/TestQueryConfig.java
 - classpath/commons-codec-1.9.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-http-0.0.12.jar
+- classpath/embulk-input-http-0.0.13.jar
 - classpath/httpclient-4.4.jar
 - classpath/httpcore-4.4.jar
 homepage: https://github.com/takumakanari/embulk-input-http