embulk-output-elasticsearch 0.3.1 → 0.4.0

data/gradlew

@@ -42,11 +42,6 @@ case "`uname`" in
     ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
 # Attempt to set APP_HOME
 # Resolve links: $0 may be a link
 PRG="$0"
@@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do
     fi
 done
 SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
+cd "`dirname \"$PRG\"`/" >/dev/null
 APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
+cd "$SAVED" >/dev/null
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
@@ -114,6 +109,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchHttpClient.java

@@ -0,0 +1,376 @@
+package org.embulk.output.elasticsearch;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import org.eclipse.jetty.client.HttpResponseException;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.config.ConfigException;
+import org.embulk.config.UserDataException;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.NodeAddressTask;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
+import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
+import org.slf4j.Logger;
+
+import javax.xml.bind.DatatypeConverter;
+
+import java.io.IOException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Random;
+
+public class ElasticsearchHttpClient
+{
+    private final Logger log;
+
+    // ALLOW_UNQUOTED_CONTROL_CHARS - Not expected but whether parser will allow JSON Strings to contain unquoted control characters
+    // FAIL_ON_UNKNOWN_PROPERTIES - Feature that determines whether encountering of unknown properties
+    private final ObjectMapper jsonMapper = new ObjectMapper()
+            .configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false)
+            .configure(com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+    // Elasticsearch maximum index byte size
+    // public static final int MAX_INDEX_NAME_BYTES = 255;
+    // @see https://github.com/elastic/elasticsearch/blob/master/core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java#L108
+    private final long maxIndexNameBytes = 255;
+    private final List<Character> inalidIndexCharaters = Arrays.asList('\\', '/', '*', '?', '"', '<', '>', '|', '#', ' ', ',');
+
+    public ElasticsearchHttpClient()
+    {
+        this.log = Exec.getLogger(getClass());
+    }
+
+    public void push(JsonNode records, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        int bulkActions = task.getBulkActions();
+        long bulkSize = task.getBulkSize();
+        // curl -xPOST localhost:9200/{index}/{type}/_bulk -d '
+        // {"index" : {}}\n
+        // {"k" : "v"}\n
+        // {"index" : {}}\n
+        // {"k" : "v2"}\n
+        // '
+        try {
+            String path = String.format("/%s/%s/_bulk", task.getIndex(), task.getType());
+            int recordSize = records.size();
+            String idColumn = task.getId().orNull();
+            if (recordSize > 0) {
+                StringBuilder sb = new StringBuilder();
+                for (JsonNode record : records) {
+                    sb.append(createIndexRequest(idColumn, record));
+
+                    String requestString = jsonMapper.writeValueAsString(record);
+                    sb.append("\n")
+                            .append(requestString)
+                            .append("\n");
+                }
+                sendRequest(path, HttpMethod.POST, task, retryHelper, sb.toString());
+            }
+        }
+        catch (JsonProcessingException ex) {
+            throw new DataException(ex);
+        }
+    }
+
+    public List<String> getIndexByAlias(String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // curl -XGET localhost:9200/_alias/{alias}
+        // No alias: 404
+        // Alias found: {"embulk_20161018-183738":{"aliases":{"embulk":{}}}}
+        List<String> indices = new ArrayList<>();
+        String path = String.format("/_alias/%s", aliasName);
+        JsonNode response = sendRequest(path, HttpMethod.GET, task, retryHelper);
+
+        Iterator it = response.fieldNames();
+        while (it.hasNext()) {
+            indices.add(it.next().toString());
+        }
+
+        return indices;
+    }
+
+    public boolean isIndexExisting(String indexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // curl -XGET localhost:9200/{index}
+        // No index: 404
+        // Index found: 200
+        try {
+            sendRequest(indexName, HttpMethod.GET, task, retryHelper);
+            return true;
+        }
+        catch (ResourceNotFoundException ex) {
+            return false;
+        }
+    }
+
+    public String generateNewIndexName(String indexName)
+    {
+        Timestamp time = Exec.getTransactionTime();
+        return indexName + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli());
+    }
+
+    public boolean isAliasExisting(String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // curl -XGET localhost:9200/_aliases  // List all aliases
+        // No aliases: {}
+        // Aliases found: {"embulk_20161018-183738":{"aliases":{"embulk":{}}}}
+        JsonNode response = sendRequest("/_aliases", HttpMethod.GET, task, retryHelper);
+        if (response.size() == 0) {
+            return false;
+        }
+        for (JsonNode index : response) {
+            if (index.has("aliases") && index.get("aliases").has(aliasName)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    // Should be called just once while Embulk transaction.
+    // Be sure to call after all exporting tasks completed
+    // This method will delete existing index
+    public void reassignAlias(String aliasName, String newIndexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        if (!isAliasExisting(aliasName, task, retryHelper)) {
+            assignAlias(newIndexName, aliasName, task, retryHelper);
+        }
+        else {
+            List<String> oldIndices = getIndexByAlias(aliasName, task, retryHelper);
+            assignAlias(newIndexName, aliasName, task, retryHelper);
+            for (String index : oldIndices) {
+                deleteIndex(index, task, retryHelper);
+            }
+        }
+    }
+
+    public String getEsVersion(PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // curl -XGET 'http://localhost:9200’
+        JsonNode response = sendRequest("", HttpMethod.GET, task, retryHelper);
+        return response.get("version").get("number").asText();
+    }
+
+    public void validateIndexOrAliasName(String index, String type)
+    {
+        for (int i = 0; i < index.length(); i++) {
+            if (inalidIndexCharaters.contains(index.charAt(i))) {
+                throw new ConfigException(String.format("%s '%s' must not contain the invalid characters " + inalidIndexCharaters.toString(), type, index));
+            }
+        }
+
+        if (!index.toLowerCase(Locale.ROOT).equals(index)) {
+            throw new ConfigException(String.format("%s '%s' must be lowercase", type, index));
+        }
+
+        if (index.startsWith("_") || index.startsWith("-") || index.startsWith("+")) {
+            throw new ConfigException(String.format("%s '%s' must not start with '_', '-', or '+'", type, index));
+        }
+
+        if (index.length() > maxIndexNameBytes) {
+            throw new ConfigException(String.format("%s name is too long, (%s > %s)", type, index.length(), maxIndexNameBytes));
+        }
+
+        if (index.equals(".") || index.equals("..")) {
+            throw new ConfigException("index must not be '.' or '..'");
+        }
+    }
+
+    private String createIndexRequest(String idColumn, JsonNode record) throws JsonProcessingException
+    {
+        // index name and type are set at path("/{index}/{type}"). So no need to set
+        if (idColumn != null && record.hasNonNull(idColumn)) {
+            // {"index" : {"_id" : "v"}}
+            Map<String, Map> indexRequest = new HashMap<>();
+
+            Map<String, JsonNode> idRequest = new HashMap<>();
+            idRequest.put("_id", record.get(idColumn));
+
+            indexRequest.put("index", idRequest);
+            return jsonMapper.writeValueAsString(indexRequest);
+        }
+        else {
+            // {"index" : {}}
+            return "{\"index\" : {}}";
+        }
+    }
+
+    private void assignAlias(String indexName, String aliasName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        try {
+            if (isIndexExisting(indexName, task, retryHelper)) {
+                if (isAliasExisting(aliasName, task, retryHelper)) {
+                    // curl -XPUT http://localhost:9200/_alias -d\
+                    // "actions" : [
+                    //   {"remove" : {"alias" : "{alias}", "index" : "{index_old}"}},
+                    //   {"add" : {"alias": "{alias}", "index": "{index_new}"}}
+                    // ]
+                    // Success: {"acknowledged":true}
+                    List<String> oldIndices = getIndexByAlias(aliasName, task, retryHelper);
+
+                    Map<String, String> newAlias = new HashMap<>();
+                    newAlias.put("alias", aliasName);
+                    newAlias.put("index", indexName);
+                    Map<String, Map> add = new HashMap<>();
+                    add.put("add", newAlias);
+
+                    Map<String, String> oldAlias = new HashMap<>();
+                    // TODO multiple alias?
+                    for (String oldIndex : oldIndices) {
+                        oldAlias.put("alias", aliasName);
+                        oldAlias.put("index", oldIndex);
+                    }
+                    Map<String, Map> remove = new HashMap<>();
+                    remove.put("remove", oldAlias);
+
+                    List<Map<String, Map>> actions = new ArrayList<>();
+                    actions.add(remove);
+                    actions.add(add);
+                    Map<String, List> rootTree = new HashMap<>();
+                    rootTree.put("actions", actions);
+
+                    String content = jsonMapper.writeValueAsString(rootTree);
+                    sendRequest("/_aliases", HttpMethod.POST, task, retryHelper, content);
+                    log.info("Reassigned alias [{}] to index[{}]", aliasName, indexName);
+                }
+                else {
+                    // curl -XPUT http://localhost:9200/{index}/_alias/{alias}
+                    // Success: {"acknowledged":true}
+                    String path = String.format("/%s/_alias/%s", indexName, aliasName);
+                    sendRequest(path, HttpMethod.PUT, task, retryHelper);
+                    log.info("Assigned alias [{}] to Index [{}]", aliasName, indexName);
+                }
+            }
+        }
+        catch (JsonProcessingException ex) {
+            throw new ConfigException(String.format("Failed to assign alias[%s] to index[%s]", aliasName, indexName));
+        }
+    }
+
+    private void deleteIndex(String indexName, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        // curl -XDELETE localhost:9200/{index}
+        // Success: {"acknowledged":true}
+        if (isIndexExisting(indexName, task, retryHelper)) {
+            sendRequest(indexName, HttpMethod.DELETE, task, retryHelper);
+            log.info("Deleted Index [{}]", indexName);
+        }
+    }
+
+    private JsonNode sendRequest(String path, final HttpMethod method, PluginTask task, Jetty92RetryHelper retryHelper)
+    {
+        return sendRequest(path, method, task, retryHelper, "");
+    }
+
+    private JsonNode sendRequest(String path, final HttpMethod method, PluginTask task, Jetty92RetryHelper retryHelper, final String content)
+    {
+        final String uri = createRequestUri(task, path);
+        final String authorizationHeader = getAuthorizationHeader(task);
+
+        try {
+            String responseBody = retryHelper.requestWithRetry(
+                    new StringJetty92ResponseEntityReader(task.getTimeoutMills()),
+                    new Jetty92SingleRequester() {
+                        @Override
+                        public void requestOnce(org.eclipse.jetty.client.HttpClient client, org.eclipse.jetty.client.api.Response.Listener responseListener)
+                        {
+                            org.eclipse.jetty.client.api.Request request = client
+                                    .newRequest(uri)
+                                    .accept("application/json")
+                                    .method(method);
+                            if (method == HttpMethod.POST) {
+                                request.content(new StringContentProvider(content), "application/json");
+                            }
+
+                            if (!authorizationHeader.isEmpty()) {
+                                request.header("Authorization", authorizationHeader);
+                            }
+                            request.send(responseListener);
+                        }
+
+                        @Override
+                        public boolean isResponseStatusToRetry(org.eclipse.jetty.client.api.Response response)
+                        {
+                            int status = response.getStatus();
+                            if (status == 429) {
+                                return true;  // Retry if 429.
+                            }
+                            return status / 100 != 4;  // Retry unless 4xx except for 429.
+                        }
+                    });
+            return parseJson(responseBody);
+        }
+        catch (HttpResponseException ex) {
+            if (ex.getMessage().startsWith("Response not 2xx: 404 Not Found")) {
+                throw new ResourceNotFoundException(ex);
+            }
+            throw ex;
+        }
+    }
+
+    private String createRequestUri(PluginTask task, String path)
+    {
+        if (!path.startsWith("/")) {
+            path = "/" + path;
+        }
+        String protocol = task.getUseSsl() ? "https" : "http";
+        String nodeAddress = getRandomNodeAddress(task);
+        return String.format("%s://%s%s", protocol, nodeAddress, path);
+    }
+
+    // Return node address (RoundRobin)
+    private String getRandomNodeAddress(PluginTask task)
+    {
+        List<NodeAddressTask> nodes = task.getNodes();
+        Random random = new Random();
+        int index = random.nextInt(nodes.size());
+        NodeAddressTask node =  nodes.get(index);
+        return node.getHost() + ":" + node.getPort();
+    }
+
+    private JsonNode parseJson(final String json) throws DataException
+    {
+        try {
+            return this.jsonMapper.readTree(json);
+        }
+        catch (IOException ex) {
+            throw new DataException(ex);
+        }
+    }
+
+    @VisibleForTesting
+    protected String getAuthorizationHeader(PluginTask task)
+    {
+        String header = "";
+        if (task.getAuthMethod() == AuthMethod.BASIC) {
+            String authString = task.getUser().get() + ":" + task.getPassword().get();
+            header = "Basic " + DatatypeConverter.printBase64Binary(authString.getBytes());
+        }
+        return header;
+    }
+
+    public class ResourceNotFoundException extends RuntimeException implements UserDataException
+    {
+        protected ResourceNotFoundException()
+        {
+        }
+
+        public ResourceNotFoundException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPlugin.java

@@ -1,586 +1,12 @@
 package org.embulk.output.elasticsearch;
 
-import java.io.IOException;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-import java.util.concurrent.TimeUnit;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
-
-import org.elasticsearch.action.bulk.BulkItemResponse;
-import org.elasticsearch.action.bulk.BulkProcessor;
-import org.elasticsearch.action.bulk.BulkRequest;
-import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.client.Requests;
-import org.elasticsearch.client.transport.NoNodeAvailableException;
-import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.cluster.metadata.AliasMetaData;
-import org.elasticsearch.cluster.metadata.AliasOrIndex;
-import org.elasticsearch.common.collect.ImmutableOpenMap;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.transport.InetSocketTransportAddress;
-import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.common.xcontent.XContentFactory;
-import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
-import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.elasticsearch.index.IndexNotFoundException;
-import org.elasticsearch.indices.InvalidAliasNameException;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.config.UserDataException;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.Exec;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.type.Types;
-import org.slf4j.Logger;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import com.google.inject.Inject;
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.embulk.base.restclient.RestClientOutputPluginBase;
 
 public class ElasticsearchOutputPlugin
-        implements OutputPlugin
+        extends RestClientOutputPluginBase<ElasticsearchOutputPluginDelegate.PluginTask>
 {
-    public interface NodeAddressTask
-            extends Task
-    {
-        @Config("host")
-        public String getHost();
-
-        @Config("port")
-        @ConfigDefault("9300")
-        public int getPort();
-    }
-
-    public interface PluginTask
-            extends Task
-    {
-        @Config("mode")
-        @ConfigDefault("\"insert\"")
-        public Mode getMode();
-
-        @Config("nodes")
-        public List<NodeAddressTask> getNodes();
-
-        @Config("cluster_name")
-        @ConfigDefault("\"elasticsearch\"")
-        public String getClusterName();
-
-        @Config("index")
-        public String getIndex();
-        public void setIndex(String indexName);
-
-        @Config("alias")
-        @ConfigDefault("null")
-        public Optional<String> getAlias();
-        public void setAlias(Optional<String> aliasName);
-
-        @Config("index_type")
-        public String getType();
-
-        @Config("id")
-        @ConfigDefault("null")
-        public Optional<String> getId();
-
-        @Config("bulk_actions")
-        @ConfigDefault("1000")
-        public int getBulkActions();
-
-        @Config("bulk_size")
-        @ConfigDefault("5242880")
-        public long getBulkSize();
-
-        @Config("concurrent_requests")
-        @ConfigDefault("5")
-        public int getConcurrentRequests();
-    }
-
-    private final Logger log;
-
-    @Inject
     public ElasticsearchOutputPlugin()
     {
-        log = Exec.getLogger(getClass());
+        super(ElasticsearchOutputPluginDelegate.PluginTask.class, new ElasticsearchOutputPluginDelegate());
     }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, Schema schema,
-                                  int processorCount, Control control)
-    {
-        final PluginTask task = config.loadConfig(PluginTask.class);
-
-        // confirm that a client can be initialized
-        try (Client client = createClient(task)) {
-            log.info(String.format("Executing plugin with '%s' mode.", task.getMode()));
-            if (task.getMode().equals(Mode.REPLACE)) {
-                task.setAlias(Optional.of(task.getIndex()));
-                task.setIndex(generateNewIndexName(task.getIndex()));
-                if (isExistsIndex(task.getAlias().orNull(), client) && !isAlias(task.getAlias().orNull(), client)) {
-                    throw new ConfigException(String.format("Invalid alias name [%s], an index exists with the same name as the alias", task.getAlias().orNull()));
-                }
-            }
-            log.info(String.format("Inserting data into index[%s]", task.getIndex()));
-            control.run(task.dump());
-
-            if (task.getMode().equals(Mode.REPLACE)) {
-                try {
-                    reAssignAlias(task.getAlias().orNull(), task.getIndex(), client);
-                } catch (IndexNotFoundException | InvalidAliasNameException e) {
-                    throw new ConfigException(e);
-                } catch (NoNodeAvailableException e) {
-                    throw new ConnectionException(e);
-                }
-            }
-        } catch (Exception e) {
-            throw Throwables.propagate(e);
-        }
-
-        ConfigDiff nextConfig = Exec.newConfigDiff();
-        return nextConfig;
-    }
-
-    @Override
-    public  ConfigDiff resume(TaskSource taskSource,
-                             Schema schema, int processorCount,
-                             OutputPlugin.Control control)
-    {
-        //  TODO
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-                        Schema schema, int processorCount,
-                        List<TaskReport> successTaskReports)
-    {}
-
-    private Client createClient(final PluginTask task)
-    {
-        //  @see http://www.elasticsearch.org/guide/en/elasticsearch/client/java-api/current/client.html
-        Settings settings = Settings.settingsBuilder()
-                .put("cluster.name", task.getClusterName())
-                .build();
-        TransportClient client = TransportClient.builder().settings(settings).build();
-        List<NodeAddressTask> nodes = task.getNodes();
-        for (NodeAddressTask node : nodes) {
-            try {
-                client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(node.getHost()), node.getPort()));
-            } catch (UnknownHostException | NoNodeAvailableException e) {
-                throw new ConnectionException(e);
-            }
-        }
-        return client;
-    }
-
-    private BulkProcessor newBulkProcessor(final PluginTask task, final Client client)
-    {
-        return BulkProcessor.builder(client, new BulkProcessor.Listener() {
-            @Override
-            public void beforeBulk(long executionId, BulkRequest request)
-            {
-                log.info("Execute {} bulk actions", request.numberOfActions());
-            }
-
-            @Override
-            public void afterBulk(long executionId, BulkRequest request, BulkResponse response)
-            {
-                if (response.hasFailures()) {
-                    long items = 0;
-                    if (log.isDebugEnabled()) {
-                        for (BulkItemResponse item : response.getItems()) {
-                            if (item.isFailed()) {
-                                items += 1;
-                                log.debug("   Error for {}/{}/{} for {} operation: {}",
-                                        item.getIndex(), item.getType(), item.getId(),
-                                        item.getOpType(), item.getFailureMessage());
-                            }
-                        }
-                    }
-                    log.warn("{} bulk actions failed: {}", items, response.buildFailureMessage());
-                } else {
-                    log.info("{} bulk actions succeeded", request.numberOfActions());
-                }
-            }
-
-            @Override
-            public void afterBulk(long executionId, BulkRequest request, Throwable failure)
-            {
-                if (failure.getClass() == NoNodeAvailableException.class) {
-                    log.error("Got the error during bulk processing", failure);
-                    throw new ConnectionException(failure);
-                } else {
-                    log.warn("Got the error during bulk processing", failure);
-                }
-            }
-        }).setBulkActions(task.getBulkActions())
-          .setBulkSize(new ByteSizeValue(task.getBulkSize()))
-          .setConcurrentRequests(task.getConcurrentRequests())
-          .build();
-    }
-
-    @Override
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema,
-                                        int processorIndex)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-        Client client = createClient(task);
-        BulkProcessor bulkProcessor = newBulkProcessor(task, client);
-        ElasticsearchPageOutput pageOutput = new ElasticsearchPageOutput(task, client, bulkProcessor);
-        pageOutput.open(schema);
-        return pageOutput;
-    }
-
-    public static class ElasticsearchPageOutput implements TransactionalPageOutput
-    {
-        private Logger log;
-
-        private Client client;
-        private BulkProcessor bulkProcessor;
-
-        private PageReader pageReader;
-        private Column idColumn;
-
-        private final String index;
-        private final String type;
-        private final String id;
-
-        public ElasticsearchPageOutput(PluginTask task, Client client, BulkProcessor bulkProcessor)
-        {
-            this.log = Exec.getLogger(getClass());
-
-            this.client = client;
-            this.bulkProcessor = bulkProcessor;
-
-            this.index = task.getIndex();
-            this.type = task.getType();
-            this.id = task.getId().orNull();
-        }
-
-        void open(final Schema schema)
-        {
-            pageReader = new PageReader(schema);
-            idColumn = (id == null) ? null : schema.lookupColumn(id);
-        }
-
-        @Override
-        public void add(Page page)
-        {
-            pageReader.setPage(page);
-
-            while (pageReader.nextRecord()) {
-                try {
-                    final XContentBuilder contextBuilder = XContentFactory.jsonBuilder().startObject(); //  TODO reusable??
-                    pageReader.getSchema().visitColumns(new ColumnVisitor() {
-                        @Override
-                        public void booleanColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), pageReader.getBoolean(column));
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-
-                        @Override
-                        public void longColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), pageReader.getLong(column));
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-
-                        @Override
-                        public void doubleColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), pageReader.getDouble(column));
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-
-                        @Override
-                        public void stringColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), pageReader.getString(column));
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-
-                        @Override
-                        public void jsonColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), pageReader.getJson(column).toJson());
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-
-                        @Override
-                        public void timestampColumn(Column column) {
-                            try {
-                                if (pageReader.isNull(column)) {
-                                    contextBuilder.nullField(column.getName());
-                                } else {
-                                    contextBuilder.field(column.getName(), new Date(pageReader.getTimestamp(column).toEpochMilli()));
-                                }
-                            } catch (IOException e) {
-                                try {
-                                    contextBuilder.nullField(column.getName());
-                                } catch (IOException ex) {
-                                    throw Throwables.propagate(ex);
-                                }
-                            }
-                        }
-                    });
-
-                    contextBuilder.endObject();
-                    bulkProcessor.add(newIndexRequest(getIdValue(idColumn)).source(contextBuilder));
-
-                } catch (ConnectionException | IOException e) {
-                    Throwables.propagate(e); //  TODO error handling
-                }
-            }
-        }
-
-        /**
-         * @param inputColumn
-         * @return
-         */
-        private String getIdValue(Column inputColumn) {
-            if (inputColumn == null) return null;
-            if (pageReader.isNull(inputColumn)) return null;
-            String idValue = null;
-            if (Types.STRING.equals(inputColumn.getType())) {
-                idValue = pageReader.getString(inputColumn);
-            } else if (Types.BOOLEAN.equals(inputColumn.getType())) {
-                idValue = pageReader.getBoolean(inputColumn) + "";
-            } else if (Types.DOUBLE.equals(inputColumn.getType())) {
-                idValue = pageReader.getDouble(inputColumn) + "";
-            } else if (Types.LONG.equals(inputColumn.getType())) {
-                idValue = pageReader.getLong(inputColumn) + "";
-            } else if (Types.JSON.equals(inputColumn.getType())) {
-                idValue = pageReader.getJson(inputColumn).toJson();
-            } else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
-                idValue = pageReader.getTimestamp(inputColumn).toString();
-            } else {
-                idValue = null;
-            }
-            return idValue;
-        }
-
-        private IndexRequest newIndexRequest(String idValue)
-        {
-            return Requests.indexRequest(index).type(type).id(idValue);
-        }
-
-        @Override
-        public void finish()
-        {
-            try {
-                bulkProcessor.flush();
-            } finally {
-                close();
-            }
-        }
-
-        @Override
-        public void close()
-        {
-            if (bulkProcessor != null) {
-                try {
-                    while (!bulkProcessor.awaitClose(3, TimeUnit.SECONDS)) {
-                        log.debug("wait for closing the bulk processing..");
-                    }
-                } catch (InterruptedException e) {
-                    Thread.currentThread().interrupt();
-                }
-                bulkProcessor = null;
-            }
-
-            if (client != null) {
-                client.close(); //  ElasticsearchException
-                client = null;
-            }
-        }
-
-        @Override
-        public void abort()
-        {
-            //  TODO do nothing
-        }
-
-        @Override
-        public TaskReport commit()
-        {
-            TaskReport report = Exec.newTaskReport();
-            //  TODO
-            return report;
-        }
-
-    }
-
-    public enum Mode
-    {
-        INSERT,
-        REPLACE;
-
-        @JsonValue
-        @Override
-        public String toString()
-        {
-            return name().toLowerCase(Locale.ENGLISH);
-        }
-
-        @JsonCreator
-        public static Mode fromString(String value)
-        {
-            switch (value) {
-                case "insert":
-                    return INSERT;
-                case "replace":
-                    return REPLACE;
-                default:
-                    throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are insert, truncate_insert, replace", value));
-            }
-        }
-    }
-
-    private void reAssignAlias(String aliasName, String newIndexName, Client client)
-            throws IndexNotFoundException, InvalidAliasNameException
-    {
-        if (!isExistsAlias(aliasName, client)) {
-            client.admin().indices().prepareAliases()
-                    .addAlias(newIndexName, aliasName)
-                    .execute().actionGet();
-            log.info(String.format("Assigned alias[%s] to index[%s]", aliasName, newIndexName));
-        } else {
-            List<String> oldIndices = getIndexByAlias(aliasName, client);
-            client.admin().indices().prepareAliases()
-                    .removeAlias(oldIndices.toArray(new String[oldIndices.size()]), aliasName)
-                    .addAlias(newIndexName, aliasName)
-                    .execute().actionGet();
-            log.info(String.format("Reassigned alias[%s] from index%s to index[%s]", aliasName, oldIndices, newIndexName));
-            for (String index : oldIndices) {
-                deleteIndex(index, client);
-            }
-        }
-    }
-
-    private void deleteIndex(String indexName, Client client)
-    {
-        client.admin().indices().delete(new DeleteIndexRequest(indexName)).actionGet();
-        log.info(String.format("Deleted Index [%s]", indexName));
-    }
-
-    private List<String> getIndexByAlias(String aliasName, Client client)
-    {
-        ImmutableOpenMap<String, List<AliasMetaData>> map = client.admin().indices().getAliases(new GetAliasesRequest(aliasName))
-                .actionGet().getAliases();
-        List<String> indices = new ArrayList<>();
-        for (ObjectObjectCursor<String, List<AliasMetaData>> c : map) {
-            indices.add(c.key);
-        }
-
-        return indices;
-    }
-
-    private boolean isExistsAlias(String aliasName, Client client)
-    {
-        return client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasAlias(aliasName);
-    }
-
-    private boolean isExistsIndex(String indexName, Client client)
-    {
-        return client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(indexName);
-    }
-
-    private boolean isAlias(String aliasName, Client client)
-    {
-        AliasOrIndex aliasOrIndex = client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().getAliasAndIndexLookup().get(aliasName);
-        return aliasOrIndex != null && aliasOrIndex.isAlias();
-    }
-
-    public String generateNewIndexName(String indexName)
-    {
-        Timestamp time = Exec.getTransactionTime();
-        return indexName + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli());
-    }
-
-    public class ConnectionException extends RuntimeException implements UserDataException
-    {
-        protected ConnectionException()
-        {
-        }
-
-        public ConnectionException(Throwable cause)
-        {
-            super(cause);
-        }
-    }
-
 }