embulk-output-elasticsearch 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 094bb8026b54c4560a6a0916600544016a15a590
-  data.tar.gz: a573ddb11e29da0a9e5e15f7816ca6136da85051
+  metadata.gz: b72c4cd366c2fa4f119b7c7c51d181363680bbca
+  data.tar.gz: 1261ff1b930b18629e625434c2952a1d63c1a1db
 SHA512:
-  metadata.gz: 386bbe3c4289f666988c70c915de20408d1961f2d346c1b328a1b716cc03ac17ab4209000d547fe144353a7b412e887231b9c44e5a3b5b3db8cacd77f877a453
-  data.tar.gz: 55dec51b6c857322ee3b6acc8844faf69a6bb9721f2aef5a321adb0ad3db6130e865dc3f638fff58826df5b441c1fcdbb5313fe6091807d182d428d44619cde4
+  metadata.gz: d2acb45304e3a79c05c6a68e83816cb157154ad587bc3032775fb67c876fba120b5a22d39b5314ff80bcd4e4ba33627a2e3586c6e56e9c1a74ff7072dd6efb4a
+  data.tar.gz: 592397a3ca552668f0fb7b31cbed40dcf6600817cd526f17731b6c3e744202e271064086900ac6808dbc750b34a79437e47d9f56cabb2b8dc0fe31c726756860

data/README.md

@@ -9,18 +9,22 @@
 
 ## Configuration
 
-- **cluster**: cluster name (string, default: 'elasticsearch')
+- **nodes**: list of nodes. nodes are pairs of host and port (list, required)
 - **index_name**: index name (string, required)
 - **index_type**: index type (string, required)
+- **doc_id_column**: document id column (string, default is null)
+- **bulk_actions**: bulk_actions (int, default is 1000)
+- **concurrent_requests**: concurrent_requests (int, default is 5)
 
 ## Example
 
 ```yaml
 out:
   type: elasticsearch
-  cluster: elasticsearch
-  index_name: embulk
-  index_type: embulk
+  node:
+  - {host: localhost, port: 9300}
+  index_name: <index name>
+  index_type: <index type>
 ```
 
 ## Build

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.4.0"

data/src/main/java/org/embulk/output/ElasticsearchOutputPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.output;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkProcessor;
@@ -10,8 +11,10 @@ import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.client.Requests;
+import org.elasticsearch.client.transport.TransportClient;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.transport.InetSocketTransportAddress;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.node.Node;
@@ -40,24 +43,32 @@ import java.util.concurrent.TimeUnit;
 public class ElasticsearchOutputPlugin
         implements OutputPlugin
 {
+    public interface NodeAddressTask
+            extends Task
+    {
+        @Config("host")
+        public String getHost();
+
+        @Config("port")
+        @ConfigDefault("9300")
+        public int getPort();
+    }
+
     public interface RunnerTask
             extends Task
     {
-        @Config("cluster")
-        @ConfigDefault("elasticsearch")
-        public String getClusterName();
+        @Config("nodes")
+        public List<NodeAddressTask> getNodes();
 
         @Config("index_name")
-        @ConfigDefault("embulk")
         public String getIndex();
 
         @Config("index_type")
-        @ConfigDefault("embulk")
         public String getIndexType();
 
-        @Config("doc_id")
+        @Config("doc_id_column")
         @ConfigDefault("null")
-        public Optional<String> getDocId();
+        public Optional<String> getIdColumn();
 
         @Config("bulk_actions")
         @ConfigDefault("1000")
@@ -83,9 +94,7 @@ public class ElasticsearchOutputPlugin
     {
         final RunnerTask task = config.loadConfig(RunnerTask.class);
 
-        try (Node node = createNode(task)) {
-            try (Client client = createClient(task, node)) {
-            }
+        try (Client client = createClient(task)) {
         }
 
         try {
@@ -113,21 +122,18 @@ public class ElasticsearchOutputPlugin
                         List<CommitReport> successCommitReports)
     { }
 
-    private Node createNode(final RunnerTask task)
+    private Client createClient(final RunnerTask task)
     {
         //  @see http://www.elasticsearch.org/guide/en/elasticsearch/client/java-api/current/client.html
         Settings settings = ImmutableSettings.settingsBuilder()
                 .classLoader(Settings.class.getClassLoader())
                 .build();
-        return NodeBuilder.nodeBuilder()
-                .clusterName(task.getClusterName())
-                .settings(settings)
-                .node();
-    }
-
-    private Client createClient(final RunnerTask task, final Node node)
-    {
-        return node.client();
+        TransportClient client = new TransportClient(settings);
+        List<NodeAddressTask> nodes = task.getNodes();
+        for (NodeAddressTask node : nodes) {
+            client.addTransportAddress(new InetSocketTransportAddress(node.getHost(), node.getPort()));
+        }
+        return client;
     }
 
     private BulkProcessor newBulkProcessor(final RunnerTask task, final Client client)
@@ -176,10 +182,9 @@ public class ElasticsearchOutputPlugin
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
 
-        Node node = createNode(task);
-        Client client = createClient(task, node);
+        Client client = createClient(task);
         BulkProcessor bulkProcessor = newBulkProcessor(task, client);
-        ElasticsearchPageOutput pageOutput = new ElasticsearchPageOutput(task, node, client, bulkProcessor);
+        ElasticsearchPageOutput pageOutput = new ElasticsearchPageOutput(task, client, bulkProcessor);
         pageOutput.open(schema);
         return pageOutput;
     }
@@ -188,7 +193,6 @@ public class ElasticsearchOutputPlugin
     {
         private Logger log;
 
-        private Node node;
         private Client client;
         private BulkProcessor bulkProcessor;
 
@@ -196,20 +200,18 @@ public class ElasticsearchOutputPlugin
 
         private final String index;
         private final String indexType;
-        private final String docId;
+        private final String docIdColumn;
 
-        ElasticsearchPageOutput(RunnerTask task, Node node, Client client,
-                                BulkProcessor bulkProcessor)
+        ElasticsearchPageOutput(RunnerTask task, Client client, BulkProcessor bulkProcessor)
         {
             this.log = Exec.getLogger(getClass());
 
-            this.node = node;
             this.client = client;
             this.bulkProcessor = bulkProcessor;
 
             this.index = task.getIndex();
             this.indexType = task.getIndexType();
-            this.docId = task.getDocId().orNull();
+            this.docIdColumn = task.getIdColumn().orNull();
         }
 
         void open(final Schema schema)
@@ -295,7 +297,7 @@ public class ElasticsearchOutputPlugin
 
         private IndexRequest newIndexRequest()
         {
-            return Requests.indexRequest(index).type(indexType).id(docId);
+            return Requests.indexRequest(index).type(indexType).id(docIdColumn);
         }
 
         @Override
@@ -326,11 +328,6 @@ public class ElasticsearchOutputPlugin
                 client.close(); //  ElasticsearchException
                 client = null;
             }
-
-            if (node != null) {
-                node.close();
-                node = null;
-            }
         }
 
         @Override

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-16 00:00:00.000000000 Z
+date: 2015-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -73,7 +73,7 @@ files:
 - src/main/resources/META-INF/services/org.embulk.spi.Extension
 - src/test/java/org/embulk/output/TestElasticsearchOutputPlugin.java
 - classpath/elasticsearch-1.4.2.jar
-- classpath/embulk-output-elasticsearch-0.1.1.jar
+- classpath/embulk-output-elasticsearch-0.1.2.jar
 - classpath/lucene-analyzers-common-4.10.2.jar
 - classpath/lucene-core-4.10.2.jar
 - classpath/lucene-grouping-4.10.2.jar