embulk-output-elasticsearch 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3be5c4aa4a859ab429375abbbb219665ea4036c0
-  data.tar.gz: 75f3d82aa911310d760f8669ab6c27ae6b8aa65d
+  metadata.gz: fee5ac83d6cdd1dd3d0f65f6b95e716f8b38389c
+  data.tar.gz: 67fc3c6af37486baa0f7dcbf05b189070d9d9e52
 SHA512:
-  metadata.gz: e606d402f49e8b7945e220e7a3fbdda72b4a86dc9a012155dc7bd2c94808ebd870b96af7fd551e8c9360e1ff4a707cb2b5f48f032ab69324347b1b431b3d007c
-  data.tar.gz: fb66e9cce399d2aa864330526524c2c4de8ed5ea48296fae0a89b7565e68cf653da3ecfc901e0317008e208714ac4321333d1962a9d4a79fd0c503814573f9a3
+  metadata.gz: 9e6bfc587cbaa9d535b78da022d7575e442a5619462ccca2833afa13b0a67a49a76acf94ae16693f38e5fe9f652d343c28eef489d06606b173fab2722f9926e1
+  data.tar.gz: 045a4f6338dbb233b6402021b1998fd4a4801d82f05bd4150ad1a5a121636b7c7a9b33ab23d1ab396247ea0e8331180036bfbb2e8ea95165493789ca511add3c

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.4 - 2017-06-16
+
+* [maintenance] Improve retry logic - Create RetryHelper instance only at sendRequest() method [#41](https://github.com/muga/embulk-output-elasticsearch/pull/41)
+
 ## 0.4.3 - 2017-06-12
 
 * [maintenance] Improve exception handling [#38](https://github.com/muga/embulk-output-elasticsearch/pull/38)

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.4.3"
+version = "0.4.4"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
@@ -27,8 +27,8 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.16"
     provided  "org.embulk:embulk-core:0.8.16"
-    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.2"
-    compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.2"
+    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
+    compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.16:tests"

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchHttpClient.java

@@ -4,9 +4,10 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
-import org.eclipse.jetty.client.HttpResponseException;
+import com.google.common.base.Throwables;
 import org.eclipse.jetty.client.util.StringContentProvider;
 import org.eclipse.jetty.http.HttpMethod;
+import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.embulk.config.ConfigException;
 import org.embulk.config.UserDataException;
 import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
@@ -15,6 +16,7 @@ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginT
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.time.Timestamp;
+import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
 import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
@@ -54,7 +56,7 @@ public class ElasticsearchHttpClient
         this.log = Exec.getLogger(getClass());
     }
 
-    public void push(JsonNode records, PluginTask task, Jetty92RetryHelper retryHelper)
+    public void push(JsonNode records, PluginTask task)
     {
         int bulkActions = task.getBulkActions();
         long bulkSize = task.getBulkSize();
@@ -78,7 +80,7 @@ public class ElasticsearchHttpClient
                             .append(requestString)
                             .append("\n");
                 }
-                sendRequest(path, HttpMethod.POST, task, retryHelper, sb.toString());
+                sendRequest(path, HttpMethod.POST, task, sb.toString());
             }
         }
         catch (JsonProcessingException ex) {
@@ -86,14 +88,14 @@ public class ElasticsearchHttpClient
         }
     }
 
-    public List<String> getIndexByAlias(String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    public List<String> getIndexByAlias(String aliasName, PluginTask task)
     {
         // curl -XGET localhost:9200/_alias/{alias}
         // No alias: 404
         // Alias found: {"embulk_20161018-183738":{"aliases":{"embulk":{}}}}
         List<String> indices = new ArrayList<>();
         String path = String.format("/_alias/%s", aliasName);
-        JsonNode response = sendRequest(path, HttpMethod.GET, task, retryHelper);
+        JsonNode response = sendRequest(path, HttpMethod.GET, task);
 
         Iterator it = response.fieldNames();
         while (it.hasNext()) {
@@ -103,13 +105,13 @@ public class ElasticsearchHttpClient
         return indices;
     }
 
-    public boolean isIndexExisting(String indexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    public boolean isIndexExisting(String indexName, PluginTask task)
     {
         // curl -XGET localhost:9200/{index}
         // No index: 404
         // Index found: 200
         try {
-            sendRequest(indexName, HttpMethod.GET, task, retryHelper);
+            sendRequest(indexName, HttpMethod.GET, task);
             return true;
         }
         catch (ResourceNotFoundException ex) {
@@ -123,12 +125,12 @@ public class ElasticsearchHttpClient
         return indexName + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli());
     }
 
-    public boolean isAliasExisting(String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    public boolean isAliasExisting(String aliasName, PluginTask task)
     {
         // curl -XGET localhost:9200/_aliases  // List all aliases
         // No aliases: {}
         // Aliases found: {"embulk_20161018-183738":{"aliases":{"embulk":{}}}}
-        JsonNode response = sendRequest("/_aliases", HttpMethod.GET, task, retryHelper);
+        JsonNode response = sendRequest("/_aliases", HttpMethod.GET, task);
         if (response.size() == 0) {
             return false;
         }
@@ -143,24 +145,24 @@ public class ElasticsearchHttpClient
     // Should be called just once while Embulk transaction.
     // Be sure to call after all exporting tasks completed
     // This method will delete existing index
-    public void reassignAlias(String aliasName, String newIndexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    public void reassignAlias(String aliasName, String newIndexName, PluginTask task)
     {
-        if (!isAliasExisting(aliasName, task, retryHelper)) {
-            assignAlias(newIndexName, aliasName, task, retryHelper);
+        if (!isAliasExisting(aliasName, task)) {
+            assignAlias(newIndexName, aliasName, task);
         }
         else {
-            List<String> oldIndices = getIndexByAlias(aliasName, task, retryHelper);
-            assignAlias(newIndexName, aliasName, task, retryHelper);
+            List<String> oldIndices = getIndexByAlias(aliasName, task);
+            assignAlias(newIndexName, aliasName, task);
             for (String index : oldIndices) {
-                deleteIndex(index, task, retryHelper);
+                deleteIndex(index, task);
             }
         }
     }
 
-    public String getEsVersion(PluginTask task, Jetty92RetryHelper retryHelper)
+    public String getEsVersion(PluginTask task)
     {
         // curl -XGET 'http://localhost:9200’
-        JsonNode response = sendRequest("", HttpMethod.GET, task, retryHelper);
+        JsonNode response = sendRequest("", HttpMethod.GET, task);
         return response.get("version").get("number").asText();
     }
 
@@ -208,18 +210,18 @@ public class ElasticsearchHttpClient
         }
     }
 
-    private void assignAlias(String indexName, String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    private void assignAlias(String indexName, String aliasName, PluginTask task)
     {
         try {
-            if (isIndexExisting(indexName, task, retryHelper)) {
-                if (isAliasExisting(aliasName, task, retryHelper)) {
+            if (isIndexExisting(indexName, task)) {
+                if (isAliasExisting(aliasName, task)) {
                     // curl -XPUT http://localhost:9200/_alias -d\
                     // "actions" : [
                     //   {"remove" : {"alias" : "{alias}", "index" : "{index_old}"}},
                     //   {"add" : {"alias": "{alias}", "index": "{index_new}"}}
                     // ]
                     // Success: {"acknowledged":true}
-                    List<String> oldIndices = getIndexByAlias(aliasName, task, retryHelper);
+                    List<String> oldIndices = getIndexByAlias(aliasName, task);
 
                     Map<String, String> newAlias = new HashMap<>();
                     newAlias.put("alias", aliasName);
@@ -243,14 +245,14 @@ public class ElasticsearchHttpClient
                     rootTree.put("actions", actions);
 
                     String content = jsonMapper.writeValueAsString(rootTree);
-                    sendRequest("/_aliases", HttpMethod.POST, task, retryHelper, content);
+                    sendRequest("/_aliases", HttpMethod.POST, task, content);
                     log.info("Reassigned alias [{}] to index[{}]", aliasName, indexName);
                 }
                 else {
                     // curl -XPUT http://localhost:9200/{index}/_alias/{alias}
                     // Success: {"acknowledged":true}
                     String path = String.format("/%s/_alias/%s", indexName, aliasName);
-                    sendRequest(path, HttpMethod.PUT, task, retryHelper);
+                    sendRequest(path, HttpMethod.PUT, task);
                     log.info("Assigned alias [{}] to Index [{}]", aliasName, indexName);
                 }
             }
@@ -260,24 +262,24 @@ public class ElasticsearchHttpClient
         }
     }
 
-    private void deleteIndex(String indexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    private void deleteIndex(String indexName, PluginTask task)
     {
         // curl -XDELETE localhost:9200/{index}
         // Success: {"acknowledged":true}
-        if (isIndexExisting(indexName, task, retryHelper)) {
-            waitSnapshot(task, retryHelper);
-            sendRequest(indexName, HttpMethod.DELETE, task, retryHelper);
+        if (isIndexExisting(indexName, task)) {
+            waitSnapshot(task);
+            sendRequest(indexName, HttpMethod.DELETE, task);
             log.info("Deleted Index [{}]", indexName);
         }
     }
 
-    private void waitSnapshot(PluginTask task, Jetty92RetryHelper retryHelper)
+    private void waitSnapshot(PluginTask task)
     {
         int maxSnapshotWaitingMills = task.getMaxSnapshotWaitingSecs() * 1000;
         long execCount = 1;
         long totalWaitingTime = 0;
         // Since only needs exponential backoff, don't need exception handling and others, I don't use Embulk RetryExecutor
-        while (isSnapshotProgressing(task, retryHelper)) {
+        while (isSnapshotProgressing(task)) {
             long sleepTime = ((long) Math.pow(2, execCount) * 1000);
             try {
                 Thread.sleep(sleepTime);
@@ -296,65 +298,67 @@ public class ElasticsearchHttpClient
         }
     }
 
-    private boolean isSnapshotProgressing(PluginTask task, Jetty92RetryHelper retryHelper)
+    private boolean isSnapshotProgressing(PluginTask task)
     {
         // https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#_snapshot_status
         // curl -XGET localhost:9200/_snapshot/_status
-        JsonNode response = sendRequest("/_snapshot/_status", HttpMethod.GET, task, retryHelper);
+        JsonNode response = sendRequest("/_snapshot/_status", HttpMethod.GET, task);
         String snapshots = response.get("snapshots").asText();
         return !snapshots.equals("");
     }
 
-    private JsonNode sendRequest(String path, final HttpMethod method, PluginTask task, Jetty92RetryHelper retryHelper)
+    private JsonNode sendRequest(String path, final HttpMethod method, PluginTask task)
     {
-        return sendRequest(path, method, task, retryHelper, "");
+        return sendRequest(path, method, task, "");
     }
 
-    private JsonNode sendRequest(String path, final HttpMethod method, final PluginTask task, Jetty92RetryHelper retryHelper, final String content)
+    private JsonNode sendRequest(String path, final HttpMethod method, final PluginTask task, final String content)
     {
         final String uri = createRequestUri(task, path);
         final String authorizationHeader = getAuthorizationHeader(task);
 
-        String responseBody = retryHelper.requestWithRetry(
-                new StringJetty92ResponseEntityReader(task.getTimeoutMills()),
-                new Jetty92SingleRequester() {
-                    @Override
-                    public void requestOnce(org.eclipse.jetty.client.HttpClient client, org.eclipse.jetty.client.api.Response.Listener responseListener)
-                    {
-                        org.eclipse.jetty.client.api.Request request = client
-                                .newRequest(uri)
-                                .accept("application/json")
-                                .method(method);
-                        if (method == HttpMethod.POST) {
-                            request.content(new StringContentProvider(content), "application/json");
+        try (Jetty92RetryHelper retryHelper = createRetryHelper(task)) {
+            String responseBody = retryHelper.requestWithRetry(
+                    new StringJetty92ResponseEntityReader(task.getTimeoutMills()),
+                    new Jetty92SingleRequester() {
+                        @Override
+                        public void requestOnce(org.eclipse.jetty.client.HttpClient client, org.eclipse.jetty.client.api.Response.Listener responseListener)
+                        {
+                            org.eclipse.jetty.client.api.Request request = client
+                                    .newRequest(uri)
+                                    .accept("application/json")
+                                    .method(method);
+                            if (method == HttpMethod.POST) {
+                                request.content(new StringContentProvider(content), "application/json");
+                            }
+
+                            if (!authorizationHeader.isEmpty()) {
+                                request.header("Authorization", authorizationHeader);
+                            }
+                            request.send(responseListener);
                         }
 
-                        if (!authorizationHeader.isEmpty()) {
-                            request.header("Authorization", authorizationHeader);
+                        @Override
+                        public boolean isExceptionToRetry(Exception exception)
+                        {
+                            return task.getId().isPresent();
                         }
-                        request.send(responseListener);
-                    }
 
-                    @Override
-                    public boolean isExceptionToRetry(Exception exception)
-                    {
-                        return task.getId().isPresent();
-                    }
-
-                    @Override
-                    public boolean isResponseStatusToRetry(org.eclipse.jetty.client.api.Response response)
-                    {
-                        int status = response.getStatus();
-                        if (status == 404) {
-                            throw new ResourceNotFoundException("Requested resource was not found");
+                        @Override
+                        public boolean isResponseStatusToRetry(org.eclipse.jetty.client.api.Response response)
+                        {
+                            int status = response.getStatus();
+                            if (status == 404) {
+                                throw new ResourceNotFoundException("Requested resource was not found");
+                            }
+                            else if (status == 429) {
+                                return true;  // Retry if 429.
+                            }
+                            return status / 100 != 4;  // Retry unless 4xx except for 429.
                         }
-                        else if (status == 429) {
-                            return true;  // Retry if 429.
-                        }
-                        return status / 100 != 4;  // Retry unless 4xx except for 429.
-                    }
-                });
-        return parseJson(responseBody);
+                    });
+            return parseJson(responseBody);
+        }
     }
 
     private String createRequestUri(PluginTask task, String path)
@@ -387,6 +391,28 @@ public class ElasticsearchHttpClient
         }
     }
 
+    private Jetty92RetryHelper createRetryHelper(PluginTask task)
+    {
+        return new Jetty92RetryHelper(
+                task.getMaximumRetries(),
+                task.getInitialRetryIntervalMillis(),
+                task.getMaximumRetryIntervalMillis(),
+                new Jetty92ClientCreator() {
+                    @Override
+                    public org.eclipse.jetty.client.HttpClient createAndStart()
+                    {
+                        org.eclipse.jetty.client.HttpClient client = new org.eclipse.jetty.client.HttpClient(new SslContextFactory());
+                        try {
+                            client.start();
+                            return client;
+                        }
+                        catch (Exception e) {
+                            throw Throwables.propagate(e);
+                        }
+                    }
+                });
+    }
+
     @VisibleForTesting
     protected String getAuthorizationHeader(PluginTask task)
     {

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPluginDelegate.java

@@ -2,10 +2,7 @@ package org.embulk.output.elasticsearch;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.embulk.base.restclient.RestClientOutputPluginDelegate;
 import org.embulk.base.restclient.RestClientOutputTaskBase;
 import org.embulk.base.restclient.jackson.JacksonServiceRequestMapper;
@@ -21,8 +18,6 @@ import org.embulk.config.TaskReport;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 
@@ -201,21 +196,19 @@ public class ElasticsearchOutputPluginDelegate
             }
         }
 
-        try (Jetty92RetryHelper retryHelper = createRetryHelper(task)) {
-            log.info(String.format("Connecting to Elasticsearch version:%s", client.getEsVersion(task, retryHelper)));
-            log.info("Executing plugin with '{}' mode.", task.getMode());
-            client.validateIndexOrAliasName(task.getIndex(), "index");
-            client.validateIndexOrAliasName(task.getType(), "index_type");
-
-            if (task.getMode().equals(Mode.REPLACE)) {
-                task.setAlias(Optional.of(task.getIndex()));
-                task.setIndex(client.generateNewIndexName(task.getIndex()));
-                if (client.isIndexExisting(task.getAlias().orNull(), task, retryHelper) && !client.isAliasExisting(task.getAlias().orNull(), task, retryHelper)) {
-                    throw new ConfigException(String.format("Invalid alias name [%s], an index exists with the same name as the alias", task.getAlias().orNull()));
-                }
+        log.info(String.format("Connecting to Elasticsearch version:%s", client.getEsVersion(task)));
+        log.info("Executing plugin with '{}' mode.", task.getMode());
+        client.validateIndexOrAliasName(task.getIndex(), "index");
+        client.validateIndexOrAliasName(task.getType(), "index_type");
+
+        if (task.getMode().equals(Mode.REPLACE)) {
+            task.setAlias(Optional.of(task.getIndex()));
+            task.setIndex(client.generateNewIndexName(task.getIndex()));
+            if (client.isIndexExisting(task.getAlias().orNull(), task) && !client.isAliasExisting(task.getAlias().orNull(), task)) {
+                throw new ConfigException(String.format("Invalid alias name [%s], an index exists with the same name as the alias", task.getAlias().orNull()));
             }
-            log.info(String.format("Inserting data into index[%s]", task.getIndex()));
         }
+        log.info(String.format("Inserting data into index[%s]", task.getIndex()));
 
         if (task.getAuthMethod() == AuthMethod.BASIC) {
             if (!task.getUser().isPresent() || !task.getPassword().isPresent()) {
@@ -237,8 +230,7 @@ public class ElasticsearchOutputPluginDelegate
     @Override  // Overridden from |RecordBufferBuildable|
     public RecordBuffer buildRecordBuffer(PluginTask task, Schema schema, int taskIndex)
     {
-        Jetty92RetryHelper retryHelper = createRetryHelper(task);
-        return new ElasticsearchRecordBuffer("records", task, retryHelper);
+        return new ElasticsearchRecordBuffer("records", task);
     }
 
     @Override
@@ -255,36 +247,11 @@ public class ElasticsearchOutputPluginDelegate
         }
 
         log.info("Insert completed. {} records", totalInserted);
-        try (Jetty92RetryHelper retryHelper = createRetryHelper(task)) {
-            // Re assign alias only when repale mode
-            if (task.getMode().equals(Mode.REPLACE)) {
-                client.reassignAlias(task.getAlias().orNull(), task.getIndex(), task, retryHelper);
-            }
+        // Re assign alias only when repale mode
+        if (task.getMode().equals(Mode.REPLACE)) {
+            client.reassignAlias(task.getAlias().orNull(), task.getIndex(), task);
         }
 
         return Exec.newConfigDiff();
     }
-
-    @VisibleForTesting
-    protected Jetty92RetryHelper createRetryHelper(PluginTask task)
-    {
-        return new Jetty92RetryHelper(
-                task.getMaximumRetries(),
-                task.getInitialRetryIntervalMillis(),
-                task.getMaximumRetryIntervalMillis(),
-                new Jetty92ClientCreator() {
-                    @Override
-                    public org.eclipse.jetty.client.HttpClient createAndStart()
-                    {
-                        org.eclipse.jetty.client.HttpClient client = new org.eclipse.jetty.client.HttpClient(new SslContextFactory());
-                        try {
-                            client.start();
-                            return client;
-                        }
-                        catch (Exception e) {
-                            throw Throwables.propagate(e);
-                        }
-                    }
-                });
-    }
 }

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchRecordBuffer.java

@@ -12,7 +12,6 @@ import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.config.TaskReport;
 import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
 import org.embulk.spi.Exec;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -28,7 +27,6 @@ public class ElasticsearchRecordBuffer
     private final long bulkActions;
     private final long bulkSize;
     private final ElasticsearchHttpClient client;
-    private final Jetty92RetryHelper retryHelper;
     private final ObjectMapper mapper;
     private final Logger log;
     private long totalCount;
@@ -36,14 +34,13 @@ public class ElasticsearchRecordBuffer
     private long requestBytes;
     private ArrayNode records;
 
-    public ElasticsearchRecordBuffer(String attributeName, PluginTask task, Jetty92RetryHelper retryHelper)
+    public ElasticsearchRecordBuffer(String attributeName, PluginTask task)
     {
         this.attributeName = attributeName;
         this.task = task;
         this.bulkActions = task.getBulkActions();
         this.bulkSize = task.getBulkSize();
         this.client = new ElasticsearchHttpClient();
-        this.retryHelper = retryHelper;
         this.mapper = new ObjectMapper()
                 .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
                 .configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
@@ -68,7 +65,7 @@ public class ElasticsearchRecordBuffer
 
             records.add(record);
             if (requestCount >= bulkActions || requestBytes >= bulkSize) {
-                client.push(records, task, retryHelper);
+                client.push(records, task);
                 if (totalCount % 10000 == 0) {
                     log.info("Inserted {} records", totalCount);
                 }
@@ -85,18 +82,23 @@ public class ElasticsearchRecordBuffer
         }
     }
 
+    @Override
+    public void finish()
+    {
+    }
+
+    @Override
+    public void close()
+    {
+    }
+
     @Override
     public TaskReport commitWithTaskReportUpdated(TaskReport taskReport)
     {
-        try {
-            if (records.size() > 0) {
-                client.push(records, task, retryHelper);
-                log.info("Inserted {} records", records.size());
-            }
-            return Exec.newTaskReport().set("inserted", totalCount);
-        }
-        finally {
-            this.retryHelper.close();
+        if (records.size() > 0) {
+            client.push(records, task);
+            log.info("Inserted {} records", records.size());
         }
+        return Exec.newTaskReport().set("inserted", totalCount);
     }
 }

data/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java

@@ -1,14 +1,10 @@
 package org.embulk.output.elasticsearch;
 
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
-import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.embulk.config.ConfigSource;
 import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
 import org.embulk.spi.Exec;
-import org.embulk.util.retryhelper.jetty92.Jetty92ClientCreator;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 
 import java.lang.reflect.Method;
 import java.util.Arrays;
@@ -61,23 +57,21 @@ public class ElasticsearchTestUtils
     public void prepareBeforeTest(PluginTask task) throws Exception
     {
         ElasticsearchHttpClient client = new ElasticsearchHttpClient();
-        try (Jetty92RetryHelper retryHelper = createRetryHelper()) {
-            Method deleteIndex = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class, Jetty92RetryHelper.class);
-            deleteIndex.setAccessible(true);
-
-            // Delete alias
-            if (client.isAliasExisting(ES_ALIAS, task, retryHelper)) {
-                deleteIndex.invoke(client, ES_ALIAS, task, retryHelper);
-            }
-
-            // Delete index
-            if (client.isIndexExisting(ES_INDEX, task, retryHelper)) {
-                deleteIndex.invoke(client, ES_INDEX, task, retryHelper);
-            }
-
-            if (client.isIndexExisting(ES_INDEX2, task, retryHelper)) {
-                deleteIndex.invoke(client, ES_INDEX2, task, retryHelper);
-            }
+        Method deleteIndex = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class);
+        deleteIndex.setAccessible(true);
+
+        // Delete alias
+        if (client.isAliasExisting(ES_ALIAS, task)) {
+            deleteIndex.invoke(client, ES_ALIAS, task);
+        }
+
+        // Delete index
+        if (client.isIndexExisting(ES_INDEX, task)) {
+            deleteIndex.invoke(client, ES_INDEX, task);
+        }
+
+        if (client.isIndexExisting(ES_INDEX2, task)) {
+            deleteIndex.invoke(client, ES_INDEX2, task);
         }
     }
 
@@ -94,7 +88,8 @@ public class ElasticsearchTestUtils
                 .set("id", ES_ID)
                 .set("bulk_actions", ES_BULK_ACTIONS)
                 .set("bulk_size", ES_BULK_SIZE)
-                .set("concurrent_requests", ES_CONCURRENT_REQUESTS);
+                .set("concurrent_requests", ES_CONCURRENT_REQUESTS)
+                .set("maximum_retries", 2);
     }
 
     public ImmutableMap<String, Object> inputConfig()
@@ -134,26 +129,4 @@ public class ElasticsearchTestUtils
         builder.add(ImmutableMap.of("name", "comment", "type", "string"));
         return builder.build();
     }
-
-    public Jetty92RetryHelper createRetryHelper()
-    {
-        return new Jetty92RetryHelper(
-                2,
-                1000,
-                32000,
-                new Jetty92ClientCreator() {
-                    @Override
-                    public org.eclipse.jetty.client.HttpClient createAndStart()
-                    {
-                        org.eclipse.jetty.client.HttpClient client = new org.eclipse.jetty.client.HttpClient(new SslContextFactory());
-                        try {
-                            client.start();
-                            return client;
-                        }
-                        catch (Exception e) {
-                            throw Throwables.propagate(e);
-                        }
-                    }
-                });
-    }
 }

data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java

@@ -7,7 +7,6 @@ import org.embulk.config.ConfigSource;
 import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
 import org.embulk.spi.Exec;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -103,53 +102,47 @@ public class TestElasticsearchHttpClient
     public void testCreateAlias() throws Exception
     {
         ElasticsearchHttpClient client = new ElasticsearchHttpClient();
-        try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
-            PluginTask task = utils.config().loadConfig(PluginTask.class);
-            // delete index
-            Method method = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class, Jetty92RetryHelper.class);
-            method.setAccessible(true);
-            method.invoke(client, "newindex", task, retryHelper);
-
-            // create index
-            Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class);
-            sendRequest.setAccessible(true);
-            String path = String.format("/%s/", ES_INDEX);
-            sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
-
-            path = String.format("/%s/", ES_INDEX2);
-            sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
-
-            // create alias
-            client.reassignAlias(ES_ALIAS, ES_INDEX, task, retryHelper);
-
-            // check alias
-            assertThat(client.isAliasExisting(ES_ALIAS, task, retryHelper), is(true));
-            assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX + "]"));
-
-            // reassign index
-            client.reassignAlias(ES_ALIAS, ES_INDEX2, task, retryHelper);
-            assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX2 + "]"));
-        }
+        PluginTask task = utils.config().loadConfig(PluginTask.class);
+        // delete index
+        Method method = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class);
+        method.setAccessible(true);
+        method.invoke(client, "newindex", task);
+
+        // create index
+        Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class);
+        sendRequest.setAccessible(true);
+        String path = String.format("/%s/", ES_INDEX);
+        sendRequest.invoke(client, path, HttpMethod.PUT, task);
+
+        path = String.format("/%s/", ES_INDEX2);
+        sendRequest.invoke(client, path, HttpMethod.PUT, task);
+
+        // create alias
+        client.reassignAlias(ES_ALIAS, ES_INDEX, task);
+
+        // check alias
+        assertThat(client.isAliasExisting(ES_ALIAS, task), is(true));
+        assertThat(client.getIndexByAlias(ES_ALIAS, task).toString(), is("[" + ES_INDEX + "]"));
+
+        // reassign index
+        client.reassignAlias(ES_ALIAS, ES_INDEX2, task);
+        assertThat(client.getIndexByAlias(ES_ALIAS, task).toString(), is("[" + ES_INDEX2 + "]"));
     }
 
     @Test
     public void testIsIndexExistingWithNonExistsIndex()
     {
         ElasticsearchHttpClient client = new ElasticsearchHttpClient();
-        try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
-            PluginTask task = utils.config().loadConfig(PluginTask.class);
-            assertThat(client.isIndexExisting("non-existing-index", task, retryHelper), is(false));
-        }
+        PluginTask task = utils.config().loadConfig(PluginTask.class);
+        assertThat(client.isIndexExisting("non-existing-index", task), is(false));
     }
 
     @Test
     public void testIsAliasExistingWithNonExistsAlias()
     {
         ElasticsearchHttpClient client = new ElasticsearchHttpClient();
-        try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
-            PluginTask task = utils.config().loadConfig(PluginTask.class);
-            assertThat(client.isAliasExisting("non-existing-alias", task, retryHelper), is(false));
-        }
+        PluginTask task = utils.config().loadConfig(PluginTask.class);
+        assertThat(client.isAliasExisting("non-existing-alias", task), is(false));
     }
 
     @Test

data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java

@@ -19,7 +19,6 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.standards.CsvParserPlugin;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -171,24 +170,22 @@ public class TestElasticsearchOutputPlugin
         output.commit();
         Thread.sleep(1500); // Need to wait until index done
 
-        try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
-            ElasticsearchHttpClient client = new ElasticsearchHttpClient();
-            Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class, String.class);
-            sendRequest.setAccessible(true);
-            String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
-            String sort = "{\"sort\" : \"id\"}";
-            JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, retryHelper, sort);
-            assertThat(response.get("hits").get("total").asInt(), is(1));
-            if (response.size() > 0) {
-                JsonNode record = response.get("hits").get("hits").get(0).get("_source");
-                assertThat(record.get("id").asInt(), is(1));
-                assertThat(record.get("account").asInt(), is(32864));
-                assertThat(record.get("time").asText(), is("2015-01-27T19:23:49.000+0000"));
-                assertThat(record.get("purchase").asText(), is("2015-01-27T00:00:00.000+0000"));
-                assertThat(record.get("flg").asBoolean(), is(true));
-                assertThat(record.get("score").asDouble(), is(123.45));
-                assertThat(record.get("comment").asText(), is("embulk"));
-            }
+        ElasticsearchHttpClient client = new ElasticsearchHttpClient();
+        Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
+        sendRequest.setAccessible(true);
+        String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
+        String sort = "{\"sort\" : \"id\"}";
+        JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
+        assertThat(response.get("hits").get("total").asInt(), is(1));
+        if (response.size() > 0) {
+            JsonNode record = response.get("hits").get("hits").get(0).get("_source");
+            assertThat(record.get("id").asInt(), is(1));
+            assertThat(record.get("account").asInt(), is(32864));
+            assertThat(record.get("time").asText(), is("2015-01-27T19:23:49.000+0000"));
+            assertThat(record.get("purchase").asText(), is("2015-01-27T00:00:00.000+0000"));
+            assertThat(record.get("flg").asBoolean(), is(true));
+            assertThat(record.get("score").asDouble(), is(123.45));
+            assertThat(record.get("comment").asText(), is("embulk"));
         }
     }
 

metadata

@@ -1,57 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-12 00:00:00.000000000 Z
+date: 2017-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: test-unit
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 3.0.2
+        version: '10.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 3.0.2
+  name: test-unit
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.2
 description: Elasticsearch output plugin is an Embulk plugin that loads records to Elasticsearch read by any input plugins. Search the input plugins by "embulk-input" keyword.
 email:
 - muga.nishizawa@gmail.com
@@ -82,9 +82,9 @@ files:
 - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
 - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
 - src/test/resources/sample_01.csv
-- classpath/embulk-base-restclient-0.5.2.jar
-- classpath/embulk-output-elasticsearch-0.4.3.jar
-- classpath/embulk-util-retryhelper-jetty92-0.5.2.jar
+- classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-output-elasticsearch-0.4.4.jar
+- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-io-9.2.14.v20151106.jar