embulk-output-elasticsearch_using_url 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 685d2edcad17b21ddc3ed1513b21d0eb60a65fbd
-  data.tar.gz: b2f159f1f335d2b702505a5ebec892fc3f3462fc
+  metadata.gz: e4a5986c1af27abb86d96bd9120cd2fee4b046e6
+  data.tar.gz: f39faffc87645aa2a74978979716893d8dca2330
 SHA512:
-  metadata.gz: f951209582d56ebbc82cf13e92e9bd52b49abf17cafd7bdea36d64fdb7f58254e103d69665f40a2f6a8282faadc7084cf1a56cc81024578fda37d49f19c47237
-  data.tar.gz: 0461770d6eedf3ab7f00204f7e31d06288768eb47a2453c20fb539695d2ff1c82fd387c476e27460ae56c5280bcabffa10453b7b43050cc47023f796de4c91aa
+  metadata.gz: f60977206fc590459fc7b6792d21b6cdbd1ff61d0e8eb217dacd13a1b132915da6b4f3c9e5ef8819de323698cc98f4b8311d5089dd6ac5968ec75ce1f4477d5e
+  data.tar.gz: 04a89192a107b50b66904dc4f97c74849c78df45c460422f4c5a75b9a8499d76c57b7488bab5d5d29f50aedbcf50495029e7477fe13056c2346e473ebbe40f8e

data/.gitignore

@@ -0,0 +1,2 @@
+/Gemfile.lock
+/pkg/

data/README.md

@@ -6,7 +6,7 @@ This plugin is forked from https://github.com/ipros-team/embulk-output-elasticse
 * **Plugin type**: output
 * **Load all or nothing**: no
 * **Resume supported**: no
-* **Cleanup supported**: no
+* **Cleanup supported**: yes
 
 ## Configuration
 

data/embulk-output-elasticsearch_using_url.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-output-elasticsearch_using_url"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["Yuma Murata"]
   spec.summary       = "Elasticsearch Using Url output plugin for Embulk"
   spec.description   = "Dumps records to Elasticsearch Using Url."

data/example/seed.yml

@@ -0,0 +1,10 @@
+in:
+  type: file
+  path_prefix: "./csv/sample_"
+out:
+  type: elasticsearch_using_url
+  mode: normal
+  nodes:
+    - url: "http://localhost:9200"
+  index: "sample"
+  index_type: "test"

data/lib/embulk/output/elasticsearch_using_url.rb

@@ -1,9 +1,12 @@
+require 'excon'
+require 'elasticsearch'
+
 module Embulk
   module Output
 
-    class ElasticsearchUsingUrl < OutputPlugin
+    class Elasticsearch < OutputPlugin
       Plugin.register_output("elasticsearch_using_url", self)
-      ENABLE_MODE = %w[normal update]
+      ENABLE_MODE = %w[normal update replace]
 
       def self.transaction(config, schema, count, &control)
         task = {
@@ -37,15 +40,66 @@ module Embulk
           client.indices.put_template name: task['before_template_name'], body: task['before_template']
         end
 
-        return self.resume(task, schema, count, &control)
-      end
-
-      def self.resume(task, schema, count, &control)
         task_reports = yield(task)
         next_config_diff = {}
         return next_config_diff
       end
 
+      def self.cleanup(task, schema, count, task_reports)
+        if task['mode'] == 'replace'
+          client = create_client(task)
+          create_aliases(client, task['index'], get_index(task))
+          delete_aliases(client, task)
+        end
+      end
+
+      def self.create_client(task)
+        return ::Elasticsearch::Client.new urls: task['nodes'].map{|v| v['url']}.join(','),
+          reload_connections: task['reload_connections'],
+          reload_on_failure: task['reload_on_failure'],
+          retry_on_failure: task['retry_on_failure'],
+          transport_options: {
+            request: { timeout: task['request_timeout'] }
+          }
+      end
+
+      def self.create_aliases(client, als, index)
+        client.indices.update_aliases body: {
+          actions: [{ add: { index: index, alias: als } }]
+        }
+        Embulk.logger.info "created alias: #{als}, index: #{index}"
+      end
+
+      def self.delete_aliases(client, task)
+        indices = client.indices.get_aliases.select { |key, value| value['aliases'].include? task['index'] }.keys
+        indices = indices.select { |index| /^#{get_index_prefix(task)}-(\d*)/ =~ index }
+        indices.each { |index|
+          if index != get_index(task)
+            client.indices.delete_alias index: index, name: task['index']
+            Embulk.logger.info "deleted alias: #{task['index']}, index: #{index}"
+            if task['delete_old_index']
+              client.indices.delete index: index
+              Embulk.logger.info "deleted index: #{index}"
+            end
+          end
+        }
+      end
+
+      def self.get_index(task)
+        task['mode'] == 'replace' ? "#{get_index_prefix(task)}-#{task['time_value']}" : task['index']
+      end
+
+      def self.get_index_prefix(task)
+        "#{task['index']}-#{task['index_type']}"
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
       def init
         @nodes = task["nodes"]
         @index_type = task["index_type"]
@@ -55,9 +109,9 @@ module Embulk
         @array_columns = task["array_columns"]
         @retry_on_failure = task["retry_on_failure"]
         @mode = task["mode"]
-        @index = task['index']
+        @index = self.class.get_index(task)
 
-        @client = create_client(task)
+        @client = self.class.create_client(task)
         @bulk_message = []
       end
 
@@ -96,16 +150,6 @@ module Embulk
 
       private
 
-      def create_client(task)
-        return ::Elasticsearch::Client.new urls: task['nodes'].map{|v| v['url']}.join(','),
-          reload_connections: task['reload_connections'],
-          reload_on_failure: task['reload_on_failure'],
-          retry_on_failure: task['retry_on_failure'],
-          transport_options: {
-            request: { timeout: task['request_timeout'] }
-          }
-      end
-
       def generate_array(record)
         result = {}
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch_using_url
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Yuma Murata
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-20 00:00:00.000000000 Z
+date: 2017-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: elasticsearch
@@ -87,11 +87,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - embulk-output-elasticsearch_using_url.gemspec
+- example/csv/sample_01.csv.gz
+- example/seed.yml
 - lib/embulk/output/elasticsearch_using_url.rb
 homepage: https://github.com/ymurata/embulk-output-elasticsearch_using_url
 licenses: