embulk-output-elasticsearch 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a3ec0cee5c598381d1fb7b9633e5b75ddd8a10b
-  data.tar.gz: c4168d8f8e4d74feea54657f95beb46c3faa424e
+  metadata.gz: 75dffd640bad902148ef5c1a3ad38ab1f3621979
+  data.tar.gz: b2f3b8a9e8573c821494d953f481ec2cd3d03c77
 SHA512:
-  metadata.gz: 3db949e32d6ec45729b717762b7a952013b40f32f2f7964415f215adeae29da4acf8fa229cffc69770dcb645bb7d80a236f135d38a38a5b76bfc3d4ff0fabc1e
-  data.tar.gz: b72a05b4607c9636febe5b67f825462d7df6a16cce257e70d952f79d873575cf055fc7c92f72d57d80dca9c66c3422e7b696ae7b89314462f988b8ef5be27586
+  metadata.gz: df79299ab2a3158c6986a6c61ad7cef7b43775d7297a5898021cf471090258887096ae66c070b5f63c8eab6d5f16b2275e65a1f7b95bd11b231cf3f982521a62
+  data.tar.gz: 1d285ccb4fd669a827c08d66c8ae5ea4bbc7097ab021c81002c14d52a46216bc20a54fa369e18f69b27f290ddc772faeba907e7c6a48166e92fc5ad9aad740a4

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.4.1 - 2017-04-21
+
+* [maintenance] Check snapshot progress status before delete index [#36](https://github.com/muga/embulk-output-elasticsearch/pull/36)
+
 ## 0.4.0 - 2017-03-28
 
 * [new feature] Support multiple Elasticsearch version [#32](https://github.com/muga/embulk-output-elasticsearch/pull/32)

data/README.md

@@ -27,6 +27,11 @@ This plugin uses HTTP/REST Client and haven't be implemented AWS authentication.
 - **bulk_actions**: Sets when to flush a new bulk request based on the number of actions currently added. (int, default is 1000)
 - **bulk_size**: Sets when to flush a new bulk request based on the size of actions currently added. (long, default is 5242880)
 - ~~**concurrent_requests**: concurrent_requests (int, default is 5)~~  Not used now. May use in the future
+- **maximum_retries** Number of maximam retry times (int, optional, default is 7)
+- **initial_retry_interval_millis** Initial interval between retries in milliseconds (int, optional, default is 1000)
+- **maximum_retry_interval_millis** Maximum interval between retries in milliseconds (int, optional, default is 120000)
+- **timeout_millis** timeout in milliseconds for HTTP client(int, optional, default is 60000)
+- **max_snapshot_waiting_secs** maximam waiting time in second when snapshot is just creating before delete index. works when `mode: replace` (int, optional, default is 1800)
 
 ### Modes
 

data/build.gradle

@@ -18,7 +18,7 @@ configurations {
     provided
 }
 
-version = "0.4.0"
+version = "0.4.1"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchHttpClient.java

@@ -265,11 +265,46 @@ public class ElasticsearchHttpClient
         // curl -XDELETE localhost:9200/{index}
         // Success: {"acknowledged":true}
         if (isIndexExisting(indexName, task, retryHelper)) {
+            waitSnapshot(task, retryHelper);
             sendRequest(indexName, HttpMethod.DELETE, task, retryHelper);
             log.info("Deleted Index [{}]", indexName);
         }
     }
 
+    private void waitSnapshot(PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        int maxSnapshotWaitingMills = task.getMaxSnapshotWaitingSecs() * 1000;
+        long execCount = 1;
+        long totalWaitingTime = 0;
+        // Since only needs exponential backoff, don't need exception handling and others, I don't use Embulk RetryExecutor
+        while (isSnapshotProgressing(task, retryHelper)) {
+            long sleepTime = ((long) Math.pow(2, execCount) * 1000);
+            try {
+                Thread.sleep(sleepTime);
+            }
+            catch (InterruptedException ex) {
+                // do nothing
+            }
+            if (execCount > 1) {
+                log.info("Waiting for snapshot completed.");
+            }
+            execCount++;
+            totalWaitingTime += sleepTime;
+            if (totalWaitingTime > maxSnapshotWaitingMills) {
+                throw new ConfigException(String.format("Waiting creating snapshot is expired. %s sec.", maxSnapshotWaitingMills));
+            }
+        }
+    }
+
+    private boolean isSnapshotProgressing(PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-snapshots.html#_snapshot_status
+        // curl -XGET localhost:9200/_snapshot/_status
+        JsonNode response = sendRequest("/_snapshot/_status", HttpMethod.GET, task, retryHelper);
+        String snapshots = response.get("snapshots").asText();
+        return !snapshots.equals("");
+    }
+
     private JsonNode sendRequest(String path, final HttpMethod method, PluginTask task, Jetty92RetryHelper retryHelper)
     {
         return sendRequest(path, method, task, retryHelper, "");

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPluginDelegate.java

@@ -126,6 +126,10 @@ public class ElasticsearchOutputPluginDelegate
         @ConfigDefault("60000")
         int getTimeoutMills();
 
+        @Config("max_snapshot_waiting_secs")
+        @ConfigDefault("1800")
+        int getMaxSnapshotWaitingSecs();
+
         @Config("time_zone")
         @ConfigDefault("\"UTC\"")
         String getTimeZone();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-29 00:00:00.000000000 Z
+date: 2017-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +83,7 @@ files:
 - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
 - src/test/resources/sample_01.csv
 - classpath/embulk-base-restclient-0.4.2.jar
-- classpath/embulk-output-elasticsearch-0.4.0.jar
+- classpath/embulk-output-elasticsearch-0.4.1.jar
 - classpath/embulk-util-retryhelper-jetty92-0.4.2.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar