embulk-output-elasticsearch_ruby 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7ca9bce994b986ee593e702147af3d22a8fb356e
+  data.tar.gz: aacbb3bd967e9f93dc27dac24506a1c060b9aef3
+SHA512:
+  metadata.gz: 05762c410db87e80d0ffe68970ed62437a8273c3daa177b530e865ab768c62f15d6940feb92e4499f529623063f09f23a6c349b253a4234f65797703e35e7ce1
+  data.tar.gz: 248d53aed9f7dbd9a4ba0c4375f50d171fc4ec37848d4e065f418df9670f9d2183eb4105ccf3c748f10a4aee3985fd59270cef85fba22abaf27386a23ba3459e

data/.gitignore

@@ -0,0 +1,7 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/vendor/
+/Gemfile.lock
+.ruby-version

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Elasticsearch Ruby output plugin for Embulk
+
+Dumps records to Elasticsearch Ruby. Elasticsearch 1.X AND 2.X AND 5.X compatible.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+
+## Configuration
+  - **nodes**: nodes (array, default: [{ 'host' => 'localhost', 'port' => 9200 }])
+    - **host**: index (string)
+    - **port**: index (integer)
+  - **request_timeout**: request_timeout (integer, default: 60)
+  - **index**: index (string, , default: 'logstash-%Y.%m.%d')
+  - **mode**: mode, normal or update or replace (string, default: normal)
+  - **reload_connections**: reload_connections (bool, default: true)
+  - **reload_on_failure**: reload_on_failure (bool, default: false)
+  - **delete_old_index**: delete_old_index (bool, default: false)
+  - **index_type**: index_type (string)
+  - **id_keys**: id_keys (array, default: nil)
+  - **id_format**: id_format (string, default: nil)
+  - **array_columns**: array_columns (array, default: nil)
+    - **name**: Array convert column. (string)
+    - **delimiter**: delimiter for split. (string)
+    - **is_integer**: to integer. (bool)
+  - **bulk_actions**: bulk_actions (integer, default: 1000)
+  - **retry_on_failure**: retry_on_failure (integer, default: 5)
+
+## Example
+
+```yaml
+out:
+  type: elasticsearch_ruby
+  nodes:
+    - {host: localhost, port: 9200}
+  index_type: page
+```
+
+## Example(update)
+
+```yaml
+out:
+  type: elasticsearch_ruby
+  nodes:
+    - {host: {{ env.ES_HOST }}, port: 9200}
+  index: crawl
+  index_type: page
+  bulk_actions: 1000
+  request_timeout: 60
+  mode: update
+  id_format: "%s"
+  id_keys:
+    - _id
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  t.warning = false
+end
+task :default => :test

data/embulk-output-elasticsearch_ruby.gemspec

@@ -0,0 +1,23 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-elasticsearch_ruby"
+  spec.version       = "0.1.1"
+  spec.authors       = ["toyama0919"]
+  spec.summary       = "Elasticsearch Ruby output plugin for Embulk. Elasticsearch 1.X AND 2.X AND 5.X compatible."
+  spec.description   = "Dumps records to Elasticsearch Ruby. Elasticsearch 1.X AND 2.X AND 5.X compatible."
+  spec.email         = ["toyama0919@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/toyama0919/embulk-output-elasticsearch_ruby"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'elasticsearch'
+  spec.add_dependency 'excon'
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'embulk', ['>= 0.8.15']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
+end

data/lib/embulk/output/elasticsearch_ruby.rb

@@ -0,0 +1,192 @@
+require 'excon'
+require 'elasticsearch'
+
+module Embulk
+  module Output
+
+    class Elasticsearch < OutputPlugin
+      Plugin.register_output("elasticsearch_ruby", self)
+      ENABLE_MODE = %w[normal update replace]
+
+      def self.transaction(config, schema, count, &control)
+        task = {
+          "nodes" => config.param("nodes", :array, default: [{ 'host' => 'localhost', 'port' => 9200 }]),
+          "request_timeout" => config.param("request_timeout", :integer, default: 60),
+          "index" => config.param("index", :string, default: 'logstash-%Y.%m.%d'),
+          "mode" => config.param("mode", :string, default: 'normal'),
+          "reload_connections" => config.param("reload_connections", :bool, default: true),
+          "reload_on_failure" => config.param("reload_on_failure", :bool, default: false),
+          "delete_old_index" => config.param("delete_old_index", :bool, default: false),
+          "index_type" => config.param("index_type", :string),
+          "id_keys" => config.param("id_keys", :array, default: nil),
+          "id_format" => config.param("id_format", :string, default: nil),
+          "array_columns" => config.param("array_columns", :array, default: nil),
+          "bulk_actions" => config.param("bulk_actions", :integer, default: 1000),
+          "retry_on_failure" => config.param("retry_on_failure", :integer, default: 5),
+        }
+        task['time_value'] = Time.now.strftime('%Y.%m.%d.%H.%M.%S')
+        task['index'] = Time.now.strftime(task['index'])
+
+        unless ENABLE_MODE.include?(task['mode'])
+          raise ConfigError.new "`mode` must be one of #{ENABLE_MODE.join(', ')}"
+        end
+        Embulk.logger.info("mode => #{task['mode']}")
+
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def self.cleanup(task, schema, count, task_reports)
+        if task['mode'] == 'replace'
+          client = create_client(task)
+          create_aliases(client, task['index'], get_index(task))
+          delete_aliases(client, task)
+        end
+      end
+
+      def self.create_client(task)
+        transport = ::Elasticsearch::Transport::Transport::HTTP::Faraday.new(
+          {
+            hosts: task['nodes'].map{ |node| Hash[node.map{ |k, v| [k.to_sym, v] }] },
+            options: {
+              reload_connections: task['reload_connections'],
+              reload_on_failure: task['reload_on_failure'],
+              retry_on_failure: task['retry_on_failure'],
+              transport_options: {
+                request: { timeout: task['request_timeout'] }
+              }
+            }
+          }
+        )
+
+        ::Elasticsearch::Client.new transport: transport
+      end
+
+      def self.create_aliases(client, als, index)
+        client.indices.update_aliases body: {
+          actions: [{ add: { index: index, alias: als } }]
+        }
+        Embulk.logger.info "created alias: #{als}, index: #{index}"
+      end
+
+      def self.delete_aliases(client, task)
+        indices = client.indices.get_aliases.select { |key, value| value['aliases'].include? task['index'] }.keys
+        indices = indices.select { |index| /^#{get_index_prefix(task)}-(\d*)/ =~ index }
+        indices.each { |index|
+          if index != get_index(task)
+            client.indices.delete_alias index: index, name: task['index']
+            Embulk.logger.info "deleted alias: #{task['index']}, index: #{index}"
+            if task['delete_old_index']
+              client.indices.delete index: index
+              Embulk.logger.info "deleted index: #{index}"
+            end
+          end
+        }
+      end
+
+      def self.get_index(task)
+        task['mode'] == 'replace' ? "#{get_index_prefix(task)}-#{task['time_value']}" : task['index']
+      end
+
+      def self.get_index_prefix(task)
+        "#{task['index']}-#{task['index_type']}"
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def init
+        @nodes = task["nodes"]
+        @index_type = task["index_type"]
+        @id_keys = task["id_keys"]
+        @id_format = task["id_format"]
+        @bulk_actions = task["bulk_actions"]
+        @array_columns = task["array_columns"]
+        @retry_on_failure = task["retry_on_failure"]
+        @mode = task["mode"]
+        @index = self.class.get_index(task)
+
+        @client = self.class.create_client(task)
+        @bulk_message = []
+      end
+
+      def close
+      end
+
+      def add(page)
+        page.each do |record|
+          hash = Hash[schema.names.zip(record)]
+          action = (@mode == 'update') ? :update : :index
+          meta = {}
+          meta[action] = { _index: @index, _type: @index_type }
+          meta[action][:_id] = generate_id(@id_format, hash, @id_keys) unless @id_keys.nil?
+          source = generate_array(hash)
+          @bulk_message << meta
+          @bulk_message << source
+          if @bulk_actions * 2 <= @bulk_message.size
+            send
+          end
+        end
+      end
+
+      def finish
+        if @bulk_message.size > 0
+          send
+        end
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+
+      private
+
+      def generate_array(record)
+        result = {}
+
+        record.each { |key, value|
+          result[key] = value
+          next if (value.nil? || !@array_columns)
+          @array_columns.each do |array_column|
+            if array_column['name'] == key
+              array_value = value.split(array_column['delimiter']).reject(&:empty?)
+              array_value = array_value.map(&:to_i) if array_column['is_integer']
+              result[key] = array_value
+            end
+          end
+        }
+        (@mode == 'update') ? {doc: result} : result
+      end
+
+      def generate_id(template, record, id_keys)
+        template % id_keys.map { |key| record[key] }
+      end
+
+      def send
+        retries = 0
+        begin
+          @client.bulk body: @bulk_message
+          Embulk.logger.info "bulk: #{@bulk_message.size/2} success."
+        rescue => e
+          if retries < @retry_on_failure
+            retries += 1
+            Embulk.logger.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
+            sleep 2**retries
+            retry
+          end
+          raise "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
+        end
+        @bulk_message.clear
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+require 'test/unit/rr'
+
+# require 'embulk/java/bootstrap'
+require 'embulk'
+Embulk.setup
+Embulk.logger = Embulk::Logger.new('/dev/null')
+
+APP_ROOT = File.expand_path('../', __dir__)

data/test/test_transaction.rb

@@ -0,0 +1,64 @@
+require_relative './helper'
+require 'embulk/output/elasticsearch_ruby'
+
+OUTPUT_ELASTICSEARCH = Embulk::Output::Elasticsearch
+
+module Embulk
+  class Output::Elasticsearch
+    class TestTransaction < Test::Unit::TestCase
+      def least_config
+        DataSource.new({
+          'nodes'     => [{ 'host' => 'localhost', 'port' => 9200 }],
+          'index_type' => 'page'
+        })
+      end
+
+      def schema
+        Schema.new([
+          Column.new({index: 0, name: 'boolean', type: :boolean}),
+          Column.new({index: 1, name: 'long', type: :long}),
+          Column.new({index: 2, name: 'double', type: :double}),
+          Column.new({index: 3, name: 'string', type: :string}),
+          Column.new({index: 4, name: 'timestamp', type: :timestamp}),
+          Column.new({index: 5, name: 'json', type: :json}),
+        ])
+      end
+
+      def processor_count
+        1
+      end
+
+      def control
+        Proc.new {|task| task_reports = [] }
+      end
+
+      def setup
+        stub(OUTPUT_ELASTICSEARCH).transaction_report { {} }
+      end
+
+      sub_test_case "normal" do
+        def test_minimum
+          config = least_config
+          OUTPUT_ELASTICSEARCH.transaction(config, schema, processor_count, &control)
+        end
+
+        def test_mode
+          config = least_config.merge('mode' => 'update')
+          OUTPUT_ELASTICSEARCH.transaction(config, schema, processor_count, &control)
+
+          config = least_config.merge('mode' => 'replace')
+          OUTPUT_ELASTICSEARCH.transaction(config, schema, processor_count, &control)
+        end
+      end
+
+      sub_test_case "error" do
+        def test_mode
+          config = least_config.merge('mode' => 'hoge')
+          assert_raise ConfigError do
+            OUTPUT_ELASTICSEARCH.transaction(config, schema, processor_count, &control)
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-elasticsearch_ruby
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- toyama0919
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-11-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: excon
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.15
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.15
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dumps records to Elasticsearch Ruby. Elasticsearch 1.X AND 2.X AND 5.X compatible.
+email:
+- toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-elasticsearch_ruby.gemspec
+- lib/embulk/output/elasticsearch_ruby.rb
+- test/helper.rb
+- test/test_transaction.rb
+homepage: https://github.com/toyama0919/embulk-output-elasticsearch_ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: Elasticsearch Ruby output plugin for Embulk. Elasticsearch 1.X AND 2.X AND 5.X compatible.
+test_files:
+- test/helper.rb
+- test/test_transaction.rb