embulk-input-elasticsearch 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9e3e0bcff8c7de3a07cab81b1689f6f2f6b5dfe5
+  data.tar.gz: a2997eb7f98cdea800403211b75342a0ff360483
+SHA512:
+  metadata.gz: bb618f0f0c4af1cdce787ce20cc38b009a4b48eba7e678f0f198c0e0195038242d8609ef42e1744077b0a51e99a8c3d62662c42e9482c4e33802d7b7ac8adeab
+  data.tar.gz: 8b43fc2e6c9afa87bb0851e2a813c9a3633104df341db9b73c74b472e3c701ce1e04e5e287f11d94b7ea2b1e11a6a0b7e2ea9c25bc70978b627765bf87ba2abf

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/vendor/
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.5.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,71 @@
+# Elasticsearch input plugin for Embulk
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+* **Guess supported**: no
+
+## Configuration
+- **nodes**: nodes (array, required)
+  - **host**: host (string, required, default: ``)
+  - **port**: port (integer, required, default: ``)
+- **queries**: query (array, required, default: ``)
+- **index**: index (string, required, default: ``)
+- **index_type**: index_type (string, default: ``)
+- **request_timeout**: request_timeout (string, default: ``)
+- **per_size**: per_size (integer, required, default: `1000`)
+- **limit_size**: limit_size (integer, default: unlimit)
+- **fields**: fields (string, required, default: ``)
+  - **name**: name (string, required, default: ``)
+  - **type**: type (string, required, default: ``)
+  - **metadata**: metadata (boolean, required, default: ``)
+  - **time_format**: time_format (string, required, default: ``)
+
+## Example
+
+```yaml
+in:
+  type: elasticsearch
+  nodes:
+    - {host: localhost, port: 9200}
+  queries:
+    - 'page_type: HP'
+    - 'page_type: GP'
+  index: crawl
+  index_type: m_corporation_page
+  request_timeout: 60
+  per_size: 1000
+  limit_size: 200000
+  fields:
+    - { name: _id, type: string, metadata: true }
+    - { name: _type, type: string, metadata: true }
+    - { name: _index, type: string, metadata: true }
+    - { name: _score, type: double, metadata: true }
+    - { name: page_type, type: string }
+    - { name: corp_name, type: string }
+    - { name: corp_key, type: string }
+    - { name: title, type: string }
+    - { name: body, type: string }
+    - { name: url, type: string }
+    - { name: employee_range, type: long }
+    - { name: m_corporation_id, type: long }
+    - { name: cg_lv1, type: json }
+    - { name: cg_lv2, type: json }
+    - { name: cg_lv3, type: json }
+```
+
+## Support Type
+* string
+* long
+* double
+* timestamp
+* json
+* boolean
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+
+task default: :build
+
+desc "Run tests"
+task :test do
+  require "test-unit"
+
+  Test::Unit::AutoRunner.run(true, './')
+end

data/embulk-input-elasticsearch.gemspec

@@ -0,0 +1,22 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-elasticsearch"
+  spec.version       = "0.1.0"
+  spec.authors       = ["toyama0919"]
+  spec.summary       = "Elasticsearch input plugin for Embulk"
+  spec.description   = "Loads records from Elasticsearch."
+  spec.email         = ["toyama0919@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/toyama0919/embulk-input-elasticsearch"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'elasticsearch'
+  spec.add_dependency 'excon'
+  spec.add_development_dependency 'embulk', ['>= 0.8.9']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'test-unit'
+end

data/lib/embulk/input/elasticsearch.rb

@@ -0,0 +1,172 @@
+require 'excon'
+require 'elasticsearch'
+
+module Embulk
+  module Input
+
+    class Elasticsearch < InputPlugin
+      Plugin.register_input("elasticsearch", self)
+
+      def self.transaction(config, &control)
+        task = {
+          "nodes" => config.param("nodes", :array),
+          "request_timeout" => config.param("request_timeout", :integer, default: 60),
+          "index" => config.param("index", :string),
+          "reload_connections" => config.param("reload_connections", :bool, default: true),
+          "reload_on_failure" => config.param("reload_on_failure", :bool, default: false),
+          "index_type" => config.param("index_type", :string, default: nil),
+          "retry_on_failure" => config.param("retry_on_failure", :integer, default: 5),
+          "per_size" => config.param("per_size", :integer, default: 1000),
+          "limit_size" => config.param("limit_size", :integer, default: nil),
+          "fields" => config.param("fields", :array, default: nil),
+          "queries" => config.param("queries", :array),
+          "sort" => config.param("sort", :hash, default: nil)
+        }
+
+        columns = []
+        task['fields'].each_with_index{ |field, i|
+          columns << Column.new(i, field['name'], field['type'].to_sym)
+        }
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        task_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def self.create_client(task)
+        transport = ::Elasticsearch::Transport::Transport::HTTP::Faraday.new(
+          {
+            hosts: task['nodes'].map{ |node| Hash[node.map{ |k, v| [k.to_sym, v] }] },
+            options: {
+              reload_connections: task['reload_connections'],
+              reload_on_failure: task['reload_on_failure'],
+              retry_on_failure: task['retry_on_failure'],
+              transport_options: {
+                request: { timeout: task['request_timeout'] }
+              }
+            }
+          }
+        )
+
+        ::Elasticsearch::Client.new transport: transport
+      end
+
+      def init
+        @client = self.class.create_client(task)
+        @index = task['index']
+        @index_type = task['index_type']
+        @queries = task['queries']
+        @per_size = task['per_size']
+        @limit_size = task['limit_size']
+        @fields = task['fields']
+        @sort = task['sort']
+      end
+
+      def run
+        @queries.each do |query|
+          query_count = 0
+          no_source_results = search(@index_type, query, 0, 0, @routing, @fields, @sort)
+          total_count = [no_source_results['hits']['total'], @limit_size].compact.min
+          while true
+            now_results_size = query_count * @per_size
+            next_results_size = (query_count + 1) * @per_size
+            size = get_size(next_results_size, now_results_size ,total_count)
+            break if size == 0
+
+            results = get_sources(search(@index_type, query, size, now_results_size, @routing, @fields, @sort), @fields)
+            results.each do |record|
+              page_builder.add(record)
+            end
+            break if last_query?(next_results_size ,total_count)
+            query_count += 1
+          end
+        end
+        page_builder.finish
+
+        task_report = {}
+        return task_report
+      end
+
+      private
+
+      def convert_value(value, field)
+        return nil if value.nil?
+        case field["type"]
+        when "string"
+          value
+        when "long"
+          value.to_i
+        when "double"
+          value.to_f
+        when "boolean"
+          if value.is_a?(TrueClass) || value.is_a?(FalseClass)
+            value
+          else
+            downcased_val = value.downcase
+            case downcased_val
+            when 'true' then true
+            when 'false' then false
+            when '1' then true
+            when '0' then false
+            else nil
+            end
+          end
+        when "timestamp"
+          Time.parse(value)
+        when "json"
+          value
+        else
+          raise "Unsupported type #{field['type']}"
+        end
+      end
+
+      def get_size(next_results_size, now_results_size ,total_count)
+        if last_query?(next_results_size ,total_count)
+          (total_count - now_results_size)
+        else
+          @per_size
+        end
+      end
+
+      def last_query?(next_results_size ,total_count)
+        next_results_size > total_count
+      end
+
+      def search(type, query, size, from, routing, fields, sort)
+        body = { from: from }
+        body[:size] = size unless size.nil?
+        if sort
+          sorts = []
+          sort.each do |k, v|
+            sorts << { k => v }
+          end
+          body[:sort] = sorts
+        end
+        body[:query] = { query_string: { query: query } } unless query.nil?
+        search_option = { index: @index, type: type, body: body }
+        search_option[:routing] = routing unless routing.nil?
+        search_option[:_source] = fields.select{ |field| !field['metadata'] }.map { |field| field['name'] }.join(',')
+        Embulk.logger.info(%Q{search_option => #{search_option}})
+        @client.search(search_option)
+      end
+
+      def get_sources(results, fields)
+        hits = results['hits']['hits']
+        hits.map { |hit|
+          result = hit['_source']
+          fields.select{ |field| field['metadata'] }.each { |field|
+            result[field['name']] = hit[field['name']]
+          }
+          @fields.map { |field|
+            convert_value(result[field['name']], field)
+          }
+        }
+      end
+    end
+  end
+end

data/test/embulk/input/test_elasticsearch.rb

@@ -0,0 +1,11 @@
+require "embulk/command/embulk_run"
+require "embulk"
+Embulk.setup
+
+require "embulk/input/elasticsearch"
+module Embulk
+  module Input
+    class ElasticsearchInputPluginTest < Test::Unit::TestCase
+    end
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-elasticsearch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- toyama0919
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: elasticsearch
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: excon
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.9
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Loads records from Elasticsearch.
+email:
+- toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-elasticsearch.gemspec
+- lib/embulk/input/elasticsearch.rb
+- test/embulk/input/test_elasticsearch.rb
+homepage: https://github.com/toyama0919/embulk-input-elasticsearch
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Elasticsearch input plugin for Embulk
+test_files:
+- test/embulk/input/test_elasticsearch.rb