embulk-output-elasticsearch 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17a82e00d22fd47a3f0921903a8b2210c0ef938c
-  data.tar.gz: accef942b8a09e8700477fc74883693702eaae0e
+  metadata.gz: 775d2c8e4628c2c21f2cfcf707ed138fbc2ab55b
+  data.tar.gz: 7956e4ddf4ec168484598f50f3a31f8305cf7780
 SHA512:
-  metadata.gz: 7cee36d9a5099ce50c92fdf760777e1a9e6cc960a9d543486d8aad10b0244a97e87e1396e8e653759f16a44ae77a5d9cb66b8be1682221d2b1337b900156c8c9
-  data.tar.gz: 75288939178b00e2d612f640c07fee9f0b86b21d1b5d17e4b15c93b2be97cb32e616a042302bf653d5aef09ce94ad790a481d6a952240d3d7700f5c10a29c1d2
+  metadata.gz: f4283bed7a28688b10d1577d0f601c9102f02b49a0d2c79df25a4a5e3ff8b83fa03300cbd6f8a251f8c65da287df6ec2ae6e81a3797958477ae86d1f37af028b
+  data.tar.gz: 4db5b4ca8f467f0c3ae77ed6fdeeea579fca25252885eea99d4fce818c61bad3a8fe7659f7fea29cdd0e1f90836ea39197a3a0f561364fbd1bd5652f1bb0bee5

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1 - 2016-02-05
+
+* [maintenance] Fix bug. Force to fail jobs if nodes down while executing [#19](https://github.com/muga/embulk-output-elasticsearch/pull/19)
+
 ## 0.2.0 - 2016-01-26
 
 * [new feature] Support Elasticsearch 2.x [#12](https://github.com/muga/embulk-output-elasticsearch/pull/12)

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.2.1"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPlugin.java

@@ -20,6 +20,7 @@ import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.transport.NoNodeAvailableException;
 import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.cluster.metadata.AliasMetaData;
 import org.elasticsearch.cluster.metadata.AliasOrIndex;
@@ -43,6 +44,7 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.config.UserDataException;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
@@ -143,6 +145,16 @@ public class ElasticsearchOutputPlugin
             }
             log.info(String.format("Inserting data into index[%s]", task.getIndex()));
             control.run(task.dump());
+
+            if (task.getMode().equals(Mode.REPLACE)) {
+                try {
+                    reAssignAlias(task.getAlias().orNull(), task.getIndex(), client);
+                } catch (IndexNotFoundException | InvalidAliasNameException e) {
+                    throw new ConfigException(e);
+                } catch (NoNodeAvailableException e) {
+                    throw new ConnectionException(e);
+                }
+            }
         } catch (Exception e) {
             throw Throwables.propagate(e);
         }
@@ -164,16 +176,7 @@ public class ElasticsearchOutputPlugin
     public void cleanup(TaskSource taskSource,
                         Schema schema, int processorCount,
                         List<TaskReport> successTaskReports)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-        if (task.getMode().equals(Mode.REPLACE)) {
-            try (Client client = createClient(task)) {
-                reAssignAlias(task.getAlias().orNull(), task.getIndex(), client);
-            } catch (IndexNotFoundException | InvalidAliasNameException e) {
-                throw new ConfigException(e);
-            }
-        }
-    }
+    {}
 
     private Client createClient(final PluginTask task)
     {
@@ -186,8 +189,8 @@ public class ElasticsearchOutputPlugin
         for (NodeAddressTask node : nodes) {
             try {
                 client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(node.getHost()), node.getPort()));
-            } catch (UnknownHostException e) {
-                Throwables.propagate(e);
+            } catch (UnknownHostException | NoNodeAvailableException e) {
+                throw new ConnectionException(e);
             }
         }
         return client;
@@ -226,7 +229,12 @@ public class ElasticsearchOutputPlugin
             @Override
             public void afterBulk(long executionId, BulkRequest request, Throwable failure)
             {
-                log.warn("Got the error during bulk processing", failure);
+                if (failure.getClass() == NoNodeAvailableException.class) {
+                    log.error("Got the error during bulk processing", failure);
+                    throw new ConnectionException(failure);
+                } else {
+                    log.warn("Got the error during bulk processing", failure);
+                }
             }
         }).setBulkActions(task.getBulkActions())
           .setBulkSize(new ByteSizeValue(task.getBulkSize()))
@@ -376,7 +384,7 @@ public class ElasticsearchOutputPlugin
                     contextBuilder.endObject();
                     bulkProcessor.add(newIndexRequest(getIdValue(idColumn)).source(contextBuilder));
 
-                } catch (IOException e) {
+                } catch (ConnectionException | IOException e) {
                     Throwables.propagate(e); //  TODO error handling
                 }
             }
@@ -543,4 +551,17 @@ public class ElasticsearchOutputPlugin
         Timestamp time = Exec.getTransactionTime();
         return indexName + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli());
     }
+
+    public class ConnectionException extends RuntimeException implements UserDataException
+    {
+        protected ConnectionException()
+        {
+        }
+
+        public ConnectionException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+
 }

data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java

@@ -168,12 +168,11 @@ public class TestElasticsearchOutputPlugin
     }
 
     @Test
-    public void testResume()
+    public void testTransaction()
     {
         ConfigSource config = config();
         Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
-        PluginTask task = config.loadConfig(PluginTask.class);
-        plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
+        plugin.transaction(config, schema, 0, new OutputPlugin.Control()
         {
             @Override
             public List<TaskReport> run(TaskSource taskSource)
@@ -181,27 +180,16 @@ public class TestElasticsearchOutputPlugin
                 return Lists.newArrayList(Exec.newTaskReport());
             }
         });
+        // no error happens
     }
 
     @Test
-    public void testTransaction()
+    public void testResume()
     {
-        ConfigSource config = Exec.newConfigSource()
-            .set("in", inputConfig())
-            .set("parser", parserConfig(schemaConfig()))
-            .set("type", "elasticsearch")
-            .set("mode", "replace")
-            .set("nodes", ES_NODES)
-            .set("cluster_name", ES_CLUSTER_NAME)
-            .set("index", ES_INDEX)
-            .set("index_type", ES_INDEX_TYPE)
-            .set("id", ES_ID)
-            .set("bulk_actions", ES_BULK_ACTIONS)
-            .set("bulk_size", ES_BULK_SIZE)
-            .set("concurrent_requests", ES_CONCURRENT_REQUESTS
-            );
+        ConfigSource config = config();
         Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
-        plugin.transaction(config, schema, 0, new OutputPlugin.Control()
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
         {
             @Override
             public List<TaskReport> run(TaskSource taskSource)
@@ -209,6 +197,15 @@ public class TestElasticsearchOutputPlugin
                 return Lists.newArrayList(Exec.newTaskReport());
             }
         });
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        ConfigSource config = config();
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
         // no error happens
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-26 00:00:00.000000000 Z
+date: 2016-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -75,7 +75,7 @@ files:
 - classpath/commons-cli-1.3.1.jar
 - classpath/compress-lzf-1.0.2.jar
 - classpath/elasticsearch-2.0.0.jar
-- classpath/embulk-output-elasticsearch-0.2.0.jar
+- classpath/embulk-output-elasticsearch-0.2.1.jar
 - classpath/HdrHistogram-2.1.6.jar
 - classpath/hppc-0.7.1.jar
 - classpath/jackson-dataformat-cbor-2.5.3.jar