embulk-input-elasticsearch 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 209e87c44e92108a80a402616ef1d7ad7db9cbd7
-  data.tar.gz: ec7efbe789ee0e2b17d922750172ba0e9f745ef2
+  metadata.gz: 6e50e12402a605f80964ad84cf29c555d6526671
+  data.tar.gz: a69bf91656bd9a569e68ee437efecf69cd03db57
 SHA512:
-  metadata.gz: 337eac65409f489536ba0e4107cdca94115530b3781b2e2e8eb90b55e3df7e46e4f79530d9776b2ee341887bde394387fe3f08850b0aa2a1c0a50b9d9ba6c613
-  data.tar.gz: 612f003dccdab158bd094b34d2853c3cd368b955b860b5a58f71ceaa3cae416c492e556acf02550ddc28aec9920485ae15fac0a8b3b78419e4bc78f31f00cd43
+  metadata.gz: 2400cf1b6273097d67d2a71f65f2984bbf3b1a79c03b1070c02aff0e1913d99be37af80e2a6d3eba0a65ef9462cee6b53d3df23ce70c6579694f4a3c278eda3e
+  data.tar.gz: a21f701e1b9695e170858ebdb16c3042cb38f6557573d03f1142b1764b2771b9c75d27056acb47e78f2a469953cecff65d2c73a2b8addd69834dc5c51b6d1eee

data/.travis.yml

@@ -2,6 +2,7 @@ language: ruby
 cache: bundler
 rvm:
   - jruby-9.0.5.0
+  - jruby-9.1.5.0
   - jruby-head
 jdk:
   - openjdk7

data/embulk-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-elasticsearch"
-  spec.version       = "0.3.2"
+  spec.version       = "0.3.3"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Elasticsearch input plugin for Embulk"
   spec.description   = "Loads records from Elasticsearch. parallel query support."

data/lib/embulk/input/elasticsearch.rb

@@ -1,7 +1,6 @@
-require 'excon'
-require 'elasticsearch'
 require_relative 'elasticsearch/connection'
 require_relative 'elasticsearch/input_thread'
+require_relative 'elasticsearch/converter'
 
 module Embulk
   module Input
@@ -52,135 +51,24 @@ module Embulk
       def init
         @queries = task['slice_queries'][@index]
         Embulk.logger.info("this thread queries => #{@queries}")
-        @client = Connection.create_client(task)
-        @index_name = task['index']
-        @index_type = task['index_type']
-        @per_size = task['per_size']
-        @limit_size = task['limit_size']
-        @fields = task['fields']
-        @sort = task['sort']
         @add_query_to_record = task['add_query_to_record']
-        @scroll = task['scroll']
-        @retry_on_failure = task['retry_on_failure']
+        @connection = Connection.new(task)
       end
 
       def run
-        search(@index_type, @per_size, @routing, @fields, @sort)
+        @queries.each do |query|
+          @connection.search_with_query(query) { |result|
+            if @add_query_to_record
+              result << query
+            end
+            page_builder.add(result)
+          }
+        end
         page_builder.finish
 
         task_report = {}
         return task_report
       end
-
-      private
-
-      def search(type, size, routing, fields, sort)
-        @queries.each do |query|
-          search_with_query(query, type, size, routing, fields, sort)
-        end
-      end
-
-      def search_with_query(query, type, size, routing, fields, sort)
-        search_option = get_search_option(type, query, size, fields, sort)
-        Embulk.logger.info("#{search_option}")
-        r = search_with_retry { @client.search(search_option) }
-        i = 0
-        get_sources(r, fields).each do |result|
-          result_proc(result, query)
-          return if @limit_size == (i += 1)
-        end
-
-        while r = (search_with_retry { @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) }) and (not r['hits']['hits'].empty?) do
-          get_sources(r, fields).each do |result|
-            result_proc(result, query)
-            return if @limit_size == (i += 1)
-          end
-        end
-      end
-
-      def search_with_retry
-        retries = 0
-        begin
-          yield if block_given?
-        rescue => e
-          if retries < @retry_on_failure
-            retries += 1
-            Embulk.logger.warn "Could not search to Elasticsearch, resetting connection and trying again. #{e.message}"
-            sleep 2**retries
-            retry
-          end
-          Embulk.logger.error "Could not search to Elasticsearch after #{retries} retries. #{e.message}"
-          raise
-        end
-      end
-
-      def result_proc(result, query)
-        if @add_query_to_record
-          result << query
-        end
-        page_builder.add(result)
-      end
-
-      def get_search_option(type, query, size, fields, sort)
-        body = { }
-        body[:query] = { query_string: { query: query } } unless query.nil?
-        if sort
-          sorts = []
-          sort.each do |k, v|
-            sorts << { k => v }
-          end
-          body[:sort] = sorts
-        else
-          body[:sort] = ["_doc"]
-        end
-        search_option = { index: @index_name, type: type, scroll: @scroll, body: body, size: size }
-        search_option[:_source] = fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
-        search_option
-      end
-
-      def get_sources(results, fields)
-        hits = results['hits']['hits']
-        hits.map { |hit|
-          result = hit['_source']
-          fields.select{ |field| field['metadata'] }.each { |field|
-            result[field['name']] = hit[field['name']]
-          }
-          @fields.map { |field|
-            convert_value(result[field['name']], field)
-          }
-        }
-      end
-
-      def convert_value(value, field)
-        return nil if value.nil?
-        case field["type"]
-        when "string"
-          value
-        when "long"
-          value.to_i
-        when "double"
-          value.to_f
-        when "boolean"
-          if value.is_a?(TrueClass) || value.is_a?(FalseClass)
-            value
-          else
-            downcased_val = value.downcase
-            case downcased_val
-            when 'true' then true
-            when 'false' then false
-            when '1' then true
-            when '0' then false
-            else nil
-            end
-          end
-        when "timestamp"
-          Time.parse(value)
-        when "json"
-          value
-        else
-          raise "Unsupported type #{field['type']}"
-        end
-      end
     end
   end
 end

data/lib/embulk/input/elasticsearch/connection.rb

@@ -1,17 +1,38 @@
+require 'excon'
+require 'elasticsearch'
+
 module Embulk
   module Input
     class Elasticsearch < InputPlugin
       class Connection
-        def self.create_client(task)
+        def initialize(task)
+          @scroll = task['scroll']
+          @index = task['index']
+          @index_type = task['index_type']
+          @size = task['per_size']
+          @fields = task['fields']
+          @sort = task['sort']
+          @limit_size = task['limit_size']
+          @retry_on_failure = task['retry_on_failure']
+          @client = create_client(
+            nodes: task['nodes'],
+            reload_connections: task['reload_connections'],
+            reload_on_failure: task['reload_on_failure'],
+            retry_on_failure: task['retry_on_failure'],
+            request_timeout: task['request_timeout']
+          )
+        end
+
+        def create_client(nodes: ,reload_connections: ,reload_on_failure: ,retry_on_failure: ,request_timeout:)
           transport = ::Elasticsearch::Transport::Transport::HTTP::Faraday.new(
             {
-              hosts: task['nodes'].map{ |node| Hash[node.map{ |k, v| [k.to_sym, v] }] },
+              hosts: nodes.map{ |node| Hash[node.map{ |k, v| [k.to_sym, v] }] },
               options: {
-                reload_connections: task['reload_connections'],
-                reload_on_failure: task['reload_on_failure'],
-                retry_on_failure: task['retry_on_failure'],
+                reload_connections: reload_connections,
+                reload_on_failure: reload_on_failure,
+                retry_on_failure: retry_on_failure,
                 transport_options: {
-                  request: { timeout: task['request_timeout'] }
+                  request: { timeout: request_timeout }
                 }
               }
             }
@@ -19,6 +40,57 @@ module Embulk
 
           ::Elasticsearch::Client.new transport: transport
         end
+
+        def search_with_query(query)
+          search_option = get_search_option(query)
+          Embulk.logger.info("#{search_option}")
+          r = search_with_retry { @client.search(search_option) }
+          i = 0
+          Converter.get_sources(r, @fields).each do |result|
+            yield(result) if block_given?
+            return if @limit_size == (i += 1)
+          end
+
+          while r = (search_with_retry { @client.scroll(scroll_id: r['_scroll_id'], scroll: @scroll) }) and (not r['hits']['hits'].empty?) do
+            Converter.get_sources(r, @fields).each do |result|
+              yield(result) if block_given?
+              return if @limit_size == (i += 1)
+            end
+          end
+        end
+
+        def search_with_retry
+          retries = 0
+          begin
+            yield if block_given?
+          rescue => e
+            if retries < @retry_on_failure
+              retries += 1
+              Embulk.logger.warn "Could not search to Elasticsearch, resetting connection and trying again. #{e.message}"
+              sleep 2**retries
+              retry
+            end
+            Embulk.logger.error "Could not search to Elasticsearch after #{retries} retries. #{e.message}"
+            raise
+          end
+        end
+
+        def get_search_option(query)
+          body = { }
+          body[:query] = { query_string: { query: query } } unless query.nil?
+          if @sort
+            sorts = []
+            @sort.each do |k, v|
+              sorts << { k => v }
+            end
+            body[:sort] = sorts
+          else
+            body[:sort] = ["_doc"]
+          end
+          search_option = { index: @index, type: @index_type, scroll: @scroll, body: body, size: @size }
+          search_option[:_source] = @fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
+          search_option
+        end
       end
     end
   end

data/lib/embulk/input/elasticsearch/converter.rb

@@ -0,0 +1,49 @@
+module Embulk
+  module Input
+    class Elasticsearch < InputPlugin
+      class Converter
+        def self.get_sources(results, fields)
+          hits = results['hits']['hits']
+          hits.map { |hit|
+            result = hit['_source']
+            fields.map { |field|
+              value = field['metadata'] ? hit[field['name']] : result[field['name']]
+              convert_value(value, field)
+            }
+          }
+        end
+
+        def self.convert_value(value, field)
+          return nil if value.nil?
+          case field["type"]
+          when "string"
+            value
+          when "long"
+            value.to_i
+          when "double"
+            value.to_f
+          when "boolean"
+            if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+              value
+            else
+              downcased_val = value.downcase
+              case downcased_val
+              when 'true' then true
+              when 'false' then false
+              when '1' then true
+              when '0' then false
+              else nil
+              end
+            end
+          when "timestamp"
+            Time.parse(value)
+          when "json"
+            value
+          else
+            raise "Unsupported type #{field['type']}"
+          end
+        end
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -9,3 +9,5 @@ Embulk.logger = Embulk::Logger.new('/dev/null')
 
 APP_ROOT = File.expand_path('../', __dir__)
 TEST_ROOT = File.expand_path(File.dirname(__FILE__))
+
+require 'embulk/input/elasticsearch'

data/test/test_converter.rb

@@ -0,0 +1,51 @@
+require_relative './helper'
+
+Elasticsearch = Embulk::Input::Elasticsearch
+
+module Embulk
+  class Input::Elasticsearch
+    class TestConverter < Test::Unit::TestCase
+
+      def startup
+      end
+
+      def shutdown
+      end
+
+      sub_test_case "get_sources" do
+        def test_normal
+          fields = [
+            {"name"=>"_id", "type"=>"string", "metadata"=>true},
+            {"name"=>"product_id", "type"=>"long"},
+            {"name"=>"title", "type"=>"string"}
+          ]
+
+          results = {
+            "_scroll_id"=>"cXVlcnlUaGVuRmV0Y2g7NTsxNzg3MjE6WlphQ3V0WDNRYmFRcS1QQ3dCb2s5UTsxNzg3MjI6WlphQ3V0WDNRYmFRcS1QQ3dCb2s5UTsxNzg3MjM6WlphQ3V0WDNRYmFRcS1QQ3dCb2s5UTsxNzg3MjU6WlphQ3V0WDNRYmFRcS1QQ3dCb2s5UTsxNzg3MjQ6WlphQ3V0WDNRYmFRcS1QQ3dCb2s5UTswOw==",
+            "took"=>41,
+            "timed_out"=>false,
+            "_shards"=>{"total"=>5, "successful"=>5, "failed"=>0},
+            "hits"=>{
+              "total"=>1,
+              "max_score"=>nil,
+              "hits"=>[
+                {
+                  "_index"=>"test_index",
+                  "_type"=>"test_type",
+                  "_id"=>"AVTCxiCuNR-BVKOgUB7R",
+                  "_score"=>nil,
+                  "_source"=>{
+                    "title"=>"dummy title",
+                    "product_id"=>1
+                  },
+                  "sort"=>[12534]
+                }
+              ]
+            }
+          }
+          assert_equal Converter.get_sources(results, fields), [["AVTCxiCuNR-BVKOgUB7R", 1, "dummy title"]]
+        end
+      end
+    end
+  end
+end

data/test/test_input_thread.rb

@@ -0,0 +1,36 @@
+require_relative './helper'
+
+Elasticsearch = Embulk::Input::Elasticsearch
+
+module Embulk
+  class Input::Elasticsearch
+    class TestTransaction < Test::Unit::TestCase
+      sub_test_case "get_slice_from_num_threads" do
+        def test_normal
+          slice = InputThread.get_slice_from_num_threads((1..10).to_a, 5)
+          assert_equal slice.size, 5
+          assert_equal slice.first.size, 2
+        end
+
+        def test_normal_same
+          slice = InputThread.get_slice_from_num_threads((1..3).to_a, 3)
+          assert_equal slice.size, 3
+          assert_equal slice.first.size, 1
+        end
+
+        def test_num_threads_over_array_size
+          slice = InputThread.get_slice_from_num_threads((1..3).to_a, 10)
+          assert_equal slice.size, 3
+          assert_equal slice.first.size, 1
+        end
+
+        def test_rest
+          slice = InputThread.get_slice_from_num_threads((1..20).to_a, 8)
+          assert_equal slice.size, 7
+          assert_equal slice.first.size, 3
+        end
+      end
+    end
+  end
+end
+

data/test/test_transaction.rb

@@ -1,5 +1,4 @@
 require_relative './helper'
-require 'embulk/input/elasticsearch'
 require 'yaml'
 
 Elasticsearch = Embulk::Input::Elasticsearch
@@ -10,33 +9,6 @@ module Embulk
       def control
         Proc.new {|task| task_reports = [] }
       end
-
-      sub_test_case "get_slice_from_num_threads" do
-        def test_normal
-          slice = InputThread.get_slice_from_num_threads((1..10).to_a, 5)
-          assert_equal slice.size, 5
-          assert_equal slice.first.size, 2
-        end
-
-        def test_normal_same
-          slice = InputThread.get_slice_from_num_threads((1..3).to_a, 3)
-          assert_equal slice.size, 3
-          assert_equal slice.first.size, 1
-        end
-
-        def test_num_threads_over_array_size
-          slice = InputThread.get_slice_from_num_threads((1..3).to_a, 10)
-          assert_equal slice.size, 3
-          assert_equal slice.first.size, 1
-        end
-
-        def test_rest
-          slice = InputThread.get_slice_from_num_threads((1..20).to_a, 8)
-          assert_equal slice.size, 7
-          assert_equal slice.first.size, 3
-        end
-      end
-
       sub_test_case "transaction" do
         def test_normal
           yaml = YAML.load(%(
@@ -75,4 +47,4 @@ module Embulk
       end
     end
   end
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-16 00:00:00.000000000 Z
+date: 2017-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -111,8 +111,11 @@ files:
 - embulk-input-elasticsearch.gemspec
 - lib/embulk/input/elasticsearch.rb
 - lib/embulk/input/elasticsearch/connection.rb
+- lib/embulk/input/elasticsearch/converter.rb
 - lib/embulk/input/elasticsearch/input_thread.rb
 - test/helper.rb
+- test/test_converter.rb
+- test/test_input_thread.rb
 - test/test_transaction.rb
 homepage: https://github.com/toyama0919/embulk-input-elasticsearch
 licenses:
@@ -140,4 +143,6 @@ specification_version: 4
 summary: Elasticsearch input plugin for Embulk
 test_files:
 - test/helper.rb
+- test/test_converter.rb
+- test/test_input_thread.rb
 - test/test_transaction.rb