embulk-input-yelp 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9f193f36dc42a127452b454bbfea5b2daf9442b
-  data.tar.gz: 39407349b04460f26c6e751a50848b67678dff57
+  metadata.gz: 7c512afdcd58e854fe161d5bf3a4515b3a82626f
+  data.tar.gz: d4763db55ffcd4f4c8bb3cee6c47956874d3b0c9
 SHA512:
-  metadata.gz: 5d67fb5ca2e51964a2f36c773647eca6f42d52b17ebacf0eba66b97d1251b17679ab604cac223f759a03c1ec34c7941935bf416f6454083a2bebff8160aaa903
-  data.tar.gz: f4bb8a326c1f6b33cf3bfc9db67dc0caef04fcc34f5c670401fe5d5ef6900fbddde462ab7c3054e0d77afbace3f8ef495a82e1c014d14abc8ab20674b4457eae
+  metadata.gz: f1508770e8b35eff5639e559c0bcb312b4490f001a270589878cf604ede3526089e8a9a0ca4c38b1691c4275e479a55d0b985d7799c10f0f105cc1cc5f6f718c
+  data.tar.gz: 0b5463a4096d0f43e06deede02cc801683dbba031d7b7da639ba5fa3b3598cb54a6aa50d708455b459b8a160853fe422656852a55fc8442eecc164a547d4f68d

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -25,7 +25,8 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.17"
     provided "org.embulk:embulk-core:0.8.17"
-    compile  "org.embulk.base.restclient:embulk-base-restclient:0.2.3"
+    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.0"
+    compile  "org.embulk.base.restclient:embulk-util-retryhelper-jaxrs:0.5.0"
     compile  "org.glassfish.jersey.core:jersey-client:2.25.1"
     testCompile "junit:junit:4.+"
 }

data/src/main/java/org/embulk/input/yelp/YelpInputPluginDelegate.java

@@ -4,16 +4,19 @@ import java.util.List;
 import java.util.Locale;
 
 import javax.ws.rs.core.Response;
+import javax.ws.rs.client.WebTarget;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
+import com.google.common.base.Optional;
 import com.google.common.collect.Ordering;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.TaskReport;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
@@ -21,16 +24,20 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.Schema;
 import org.embulk.spi.type.Types;
 
+import org.embulk.base.restclient.DefaultServiceDataSplitter;
 import org.embulk.base.restclient.RestClientInputPluginDelegate;
 import org.embulk.base.restclient.RestClientInputTaskBase;
+import org.embulk.base.restclient.ServiceDataSplitter;
 import org.embulk.base.restclient.jackson.JacksonJsonPointerValueLocator;
 import org.embulk.base.restclient.jackson.JacksonServiceRecord;
 import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
 import org.embulk.base.restclient.jackson.StringJsonParser;
 import org.embulk.base.restclient.record.RecordImporter;
-import org.embulk.base.restclient.request.RetryHelper;
-import org.embulk.base.restclient.request.SingleRequester;
-import org.embulk.base.restclient.request.StringResponseEntityReader;
+
+import org.embulk.util.retryhelper.jaxrs.JAXRSClientCreator;
+import org.embulk.util.retryhelper.jaxrs.JAXRSRetryHelper;
+import org.embulk.util.retryhelper.jaxrs.JAXRSSingleRequester;
+import org.embulk.util.retryhelper.jaxrs.StringJAXRSResponseEntityReader;
 
 import org.slf4j.Logger;
 
@@ -44,7 +51,16 @@ public class YelpInputPluginDelegate
         public String getAccessToken();
 
         @Config("location")
-        public String getLocation();
+        @ConfigDefault("null")
+        public Optional<String> getLocation();
+
+        @Config("latitude")
+        @ConfigDefault("null")
+        public Optional<String> getLatitude();
+
+        @Config("longitude")
+        @ConfigDefault("null")
+        public Optional<String> getLongitude();
 
         @Config("maximum_retries")
         @ConfigDefault("7")
@@ -61,9 +77,19 @@ public class YelpInputPluginDelegate
 
     private final StringJsonParser jsonParser = new StringJsonParser();
 
-    @Override  // Overridden from |TaskValidatable|
-    public void validateTask(PluginTask task)
+    @Override  // Overridden from |InputTaskValidatable|
+    public void validateInputTask(PluginTask task)
     {
+        if (!task.getLocation().isPresent() &&
+            !(task.getLatitude().isPresent() && task.getLongitude().isPresent())) {
+            throw new ConfigException("'location' or 'latitude'/'longitude' are required.");
+        }
+    }
+
+    @Override  // Overridden from |InputTaskValidatable|
+    public ServiceDataSplitter buildServiceDataSplitter(PluginTask task)
+    {
+        return new DefaultServiceDataSplitter();
     }
 
     @Override  // Overridden from |ServiceResponseMapperBuildable|
@@ -95,7 +121,6 @@ public class YelpInputPluginDelegate
 
     @Override  // Overridden from |ServiceDataIngestable|
     public TaskReport ingestServiceData(final PluginTask task,
-                                        RetryHelper retryHelper,
                                         RecordImporter recordImporter,
                                         int taskIndex,
                                         PageBuilder pageBuilder)
@@ -103,11 +128,25 @@ public class YelpInputPluginDelegate
         TaskReport report = Exec.newTaskReport();
         for (int offset = 0; offset < 1000; ) {
             final int limit = Exec.isPreview() ? 5 : (Ordering.natural().min(50, 1000 - offset));
-            String content = fetchFromYelp(retryHelper,
-                                           task.getAccessToken(),
-                                           task.getLocation(),
-                                           limit,
-                                           offset);
+            final String content;
+            try (JAXRSRetryHelper retryHelper = new JAXRSRetryHelper(
+                     task.getMaximumRetries(),
+                     task.getInitialRetryIntervalMillis(),
+                     task.getMaximumRetryIntervalMillis(),
+                     new JAXRSClientCreator() {
+                         @Override
+                         public javax.ws.rs.client.Client create() {
+                             return javax.ws.rs.client.ClientBuilder.newBuilder().build();
+                         }
+                     })) {
+                content = fetchFromYelp(retryHelper,
+                                        task.getAccessToken(),
+                                        task.getLocation(),
+                                        task.getLatitude(),
+                                        task.getLongitude(),
+                                        limit,
+                                        offset);
+            }
             ArrayNode records = extractArrayField(content);
 
             for (JsonNode record : records) {
@@ -151,23 +190,34 @@ public class YelpInputPluginDelegate
         }
     }
 
-    private String fetchFromYelp(RetryHelper retryHelper,
+    private String fetchFromYelp(JAXRSRetryHelper retryHelper,
                                  final String bearerToken,
-                                 final String location,
+                                 final Optional<String> location,
+                                 final Optional<String> latitude,
+                                 final Optional<String> longitude,
                                  final int limit,
                                  final int offset)
     {
         return retryHelper.requestWithRetry(
-            new StringResponseEntityReader(),
-            new SingleRequester() {
+            new StringJAXRSResponseEntityReader(),
+            new JAXRSSingleRequester() {
                 @Override
                 public Response requestOnce(javax.ws.rs.client.Client client)
                 {
-                    Response response = client
+                    WebTarget webTarget = client
                         .target("https://api.yelp.com/v3/businesses/search")
-                        .queryParam("location", location)
                         .queryParam("limit", String.valueOf(limit))
-                        .queryParam("offset", String.valueOf(offset))
+                        .queryParam("offset", String.valueOf(offset));
+                    if (location.isPresent()) {
+                        webTarget = webTarget.queryParam("location", location.get());
+                    } else if (latitude.isPresent() && longitude.isPresent()) {
+                        webTarget = webTarget.queryParam("latitude", latitude.get())
+                                             .queryParam("longitude", longitude.get());
+                    } else {
+                        throw new ConfigException(
+                            "FATAL: 'location' or 'latitude'/'longitude' are required.");
+                    }
+                    Response response = webTarget
                         .request()
                         .header("Authorization", "Bearer " + bearerToken)
                         .get();
@@ -186,30 +236,5 @@ public class YelpInputPluginDelegate
             });
     }
 
-    @Override  // Overridden from |ClientCreatable|
-    public javax.ws.rs.client.Client createClient(PluginTask task)
-    {
-        // TODO(dmikurube): Configure org.glassfish.jersey.client.ClientProperties.CONNECT_TIMEOUT and READ_TIMEOUT.
-        return javax.ws.rs.client.ClientBuilder.newBuilder().build();
-    }
-
-    @Override  // Overridden from |RetryConfigurable|
-    public int configureMaximumRetries(PluginTask task)
-    {
-        return task.getMaximumRetries();
-    }
-
-    @Override  // Overridden from |RetryConfigurable|
-    public int configureInitialRetryIntervalMillis(PluginTask task)
-    {
-        return task.getInitialRetryIntervalMillis();
-    }
-
-    @Override  // Overridden from |RetryConfigurable|
-    public int configureMaximumRetryIntervalMillis(PluginTask task)
-    {
-        return task.getMaximumRetryIntervalMillis();
-    }
-
     private final Logger logger = Exec.getLogger(YelpInputPluginDelegate.class);
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-yelp
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Dai MIKURUBE
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-04 00:00:00.000000000 Z
+date: 2017-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,8 +60,9 @@ files:
 - src/main/java/org/embulk/input/yelp/YelpInputPluginDelegate.java
 - src/test/java/org/embulk/input/yelp/TestYelpInputPlugin.java
 - classpath/aopalliance-repackaged-2.5.0-b32.jar
-- classpath/embulk-base-restclient-0.2.3.jar
-- classpath/embulk-input-yelp-0.1.0.jar
+- classpath/embulk-base-restclient-0.5.0.jar
+- classpath/embulk-input-yelp-0.2.0.jar
+- classpath/embulk-util-retryhelper-jaxrs-0.5.0.jar
 - classpath/hk2-api-2.5.0-b32.jar
 - classpath/hk2-locator-2.5.0-b32.jar
 - classpath/hk2-utils-2.5.0-b32.jar