metacrunch-elasticsearch 3.0.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c8e449f5c84fd5b22946dc75c52b1693830e6c4
-  data.tar.gz: b698c9cc5904719a37d6902e4cee9528ea03577e
+  metadata.gz: 7fcdad44003a195fca6541d8c9156523dcde02ad
+  data.tar.gz: 964c93ec7c91b54ea76bf338c5997920546f3840
 SHA512:
-  metadata.gz: 396ae218ce4e493aa8d96bc7a5061e5941a65ea56c9d8cd791a9bcaeb0de11f5f83fc1df5f97379f967d969c86626b084dd9d39ed6fbeb3e6dbed7aacf88f2ae
-  data.tar.gz: 4254f096d00cb3d113e43f5b5b9e215699002972f6661a335a1c4683bdcaf76bcbe12128d2a45b30e8a4bb5043138a16887f6c049d6c52db9fe3e1af23e5cebe
+  metadata.gz: 29bdc05cebb697ee8fbf6a434d2ad3861cf205c0a35324a77f1c1ef145599c72032f4adf2e5cc6e2cb395ac5bc744493942f87ddeb6f0310b342ac8285c04074
+  data.tar.gz: 60afb62d71c16d17404b5fd2b1bf735bce2011398a9bc014f0cb2c0e398df13d668c1601b9caa3e6beda1ed3f0d946da4bd37c0a59548159410182e86d3faf12

data/Gemfile

@@ -3,19 +3,15 @@ source "https://rubygems.org"
 gemspec
 
 group :development do
-  gem "bundler",      ">= 1.7"
-  gem "rake",         ">= 11.1"
-  gem "rspec",        ">= 3.0.0",  "< 4.0.0"
-  gem "simplecov",    ">= 0.11.0"
+  gem "bundler", ">= 1.15"
+  gem "rake",    ">= 12.1"
+  gem "rspec",   ">= 3.5.0", "< 4.0.0"
 
   if !ENV["CI"]
-    gem "hashdiff",   ">= 0.3.0", platform: :ruby
-    gem "pry-byebug", ">= 3.3.0", platform: :ruby
-    gem "pry-rescue", ">= 1.4.2", platform: :ruby
-    gem "pry-state",  ">= 0.1.7", platform: :ruby
+    gem "pry-byebug", ">= 3.5.0"
   end
 end
 
 group :test do
-  gem "codeclimate-test-reporter", ">= 0.5.0", require: nil
+  gem "simplecov", ">= 0.15.0"
 end

data/Rakefile

@@ -1,5 +1,5 @@
-require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require "bundler/gem_tasks"
 
 RSpec::Core::RakeTask.new(:spec)
 

data/bin/console

@@ -1,11 +1,14 @@
 #!/usr/bin/env ruby
-
 require "bundler/setup"
-require "metascrunch/elasticsearch"
+require "metacrunch/elasticsearch"
 
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
+begin
+  require "pry"
+rescue LoadError ; end
 
-# (If you use this, don't forget to add pry to your Gemfile!)
-require "pry"
-Pry.start
+if defined?(Pry)
+  Pry.start
+else
+  require "irb"
+  IRB.start
+end

data/lib/metacrunch/elasticsearch.rb

@@ -2,16 +2,7 @@ require "elasticsearch"
 
 module Metacrunch
   module Elasticsearch
-    require_relative "./elasticsearch/index_creator"
-    require_relative "./elasticsearch/indexer"
-    require_relative "./elasticsearch/reader"
-    require_relative "./elasticsearch/searcher"
-    require_relative "./elasticsearch/uri"
-    require_relative "./elasticsearch/writer"
-
-    #
-    # error class are inline to not clutter source files unnecessarily
-    #
-    class IndexAlreadyExistsError < StandardError; end
+    require_relative "elasticsearch/destination"
+    require_relative "elasticsearch/source"
   end
 end

data/lib/metacrunch/elasticsearch/destination.rb

@@ -0,0 +1,47 @@
+require "metacrunch/elasticsearch"
+
+module Metacrunch
+  class Elasticsearch::Destination
+
+    DEFAULT_OPTIONS = {
+      raise_on_result_errors: false,
+      result_callback: nil,
+      bulk_options: {}
+    }
+
+    def initialize(elasticsearch_client, options = {})
+      @client = elasticsearch_client
+      @options = DEFAULT_OPTIONS.deep_merge(options)
+    end
+
+    def write(data)
+      return if data.blank?
+
+      # Call elasticsearch bulk api
+      bulk_options = @options[:bulk_options]
+      bulk_options[:body] = data
+      result = @client.bulk(bulk_options)
+
+      # Raise an exception if one of the results produced an error and the user wants to know about it
+      raise DestinationError.new(errors: result["errors"]) if result["errors"] && @options[:raise_on_result_errors]
+
+      # if the user provided a callback proc, call it
+      @options[:result_callback].call(result) if @options[:result_callback]&.respond_to?(:call)
+    end
+
+    def close
+      # noop
+    end
+
+  end
+
+  class Elasticsearch::DestinationError < StandardError
+
+    attr_reader :errors
+
+    def initialize(msg = nil, errors:)
+      @errors = errors
+    end
+
+  end
+end

data/lib/metacrunch/elasticsearch/source.rb

@@ -0,0 +1,56 @@
+require "metacrunch/elasticsearch"
+
+module Metacrunch
+  class Elasticsearch::Source
+
+    DEFAULT_OPTIONS = {
+      total_hits_callback: nil,
+      search_options: {
+        size: 100,
+        scroll: "1m",
+        sort: ["_doc"]
+      }
+    }
+
+    def initialize(elasticsearch_client, options = {})
+      @client = elasticsearch_client
+      @options = DEFAULT_OPTIONS.deep_merge(options)
+    end
+
+    def each(&block)
+      return enum_for(__method__) unless block_given?
+
+      # Perform search request and yield the first results if any
+      search_options = @options[:search_options]
+      result = @client.search(search_options)
+      call_total_hits_callback(result)
+      yield_hits(result, &block)
+
+      # Scroll over the rest of result set and yield the results until the set is empty.
+      while (
+        # Note: semantic of 'and' is important here. Do not use '&&'.
+        result = @client.scroll(scroll_id: result["_scroll_id"], scroll: search_options[:scroll]) and result["hits"]["hits"].present?
+      ) do
+        yield_hits(result, &block)
+      end
+    ensure
+      # Clear scroll to free up resources.
+      @client.clear_scroll(scroll_id: result["_scroll_id"]) if result
+    end
+
+  private
+
+    def call_total_hits_callback(result)
+      if @options[:total_hits_callback]&.respond_to?(:call) && result["hits"]["total"]
+        @options[:total_hits_callback].call(result["hits"]["total"])
+      end
+    end
+
+    def yield_hits(result, &block)
+      result["hits"]["hits"].each do |hit|
+        yield(hit)
+      end
+    end
+
+  end
+end

data/lib/metacrunch/elasticsearch/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module Elasticsearch
-    VERSION = "3.0.0"
+    VERSION = "4.0.0"
   end
 end

data/metacrunch-elasticsearch.gemspec

@@ -6,17 +6,16 @@ require "metacrunch/elasticsearch/version"
 Gem::Specification.new do |spec|
   spec.name          = "metacrunch-elasticsearch"
   spec.version       = Metacrunch::Elasticsearch::VERSION
-  spec.authors       = ["René Sprotte", "Michael Sievers"]
-  spec.summary       = %q{Metacrunch elasticsearch package}
+  spec.authors       = ["René Sprotte"]
+  spec.summary       = %q{Elasticsearch package for the metacrunch ETL toolkit.}
   spec.homepage      = "http://github.com/ubpb/metacrunch-elasticsearch"
-  spec.licenses      = "MIT"
+  spec.license       = "MIT"
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "activesupport", ">= 4.0.0"
-  spec.add_dependency "elasticsearch", "~> 1.0"
+  spec.add_dependency "activesupport", ">= 5.1.0"
+  spec.add_dependency "elasticsearch", ">= 5.0.4"
 end
 

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 4.0.0
 platform: ruby
 authors:
 - René Sprotte
-- Michael Sievers
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2016-05-18 00:00:00.000000000 Z
+date: 2017-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -17,28 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 5.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.0.0
+        version: 5.1.0
 - !ruby/object:Gem::Dependency
   name: elasticsearch
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 5.0.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 5.0.4
 description: 
 email: 
 executables: []
@@ -47,23 +46,14 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
 - License.txt
 - Rakefile
-- Readme.md
 - bin/console
-- bin/setup
 - lib/metacrunch/elasticsearch.rb
-- lib/metacrunch/elasticsearch/client_factory.rb
-- lib/metacrunch/elasticsearch/index_creator.rb
-- lib/metacrunch/elasticsearch/indexer.rb
-- lib/metacrunch/elasticsearch/options_helpers.rb
-- lib/metacrunch/elasticsearch/reader.rb
-- lib/metacrunch/elasticsearch/searcher.rb
-- lib/metacrunch/elasticsearch/uri.rb
+- lib/metacrunch/elasticsearch/destination.rb
+- lib/metacrunch/elasticsearch/source.rb
 - lib/metacrunch/elasticsearch/version.rb
-- lib/metacrunch/elasticsearch/writer.rb
 - metacrunch-elasticsearch.gemspec
 homepage: http://github.com/ubpb/metacrunch-elasticsearch
 licenses:
@@ -85,8 +75,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
-summary: Metacrunch elasticsearch package
+summary: Elasticsearch package for the metacrunch ETL toolkit.
 test_files: []

data/.travis.yml

@@ -1,5 +0,0 @@
-language: ruby
-rvm:
-  - "2.0"
-  - "2.1"
-  - "2.2"

data/Readme.md

@@ -1,3 +0,0 @@
-# metacrunch-elasticsearch
-
-Dokumentation folgt in Kürze...

data/bin/setup

@@ -1,7 +0,0 @@
-#!/bin/bash
-set -euo pipefail
-IFS=$'\n\t'
-
-bundle install
-
-# Do any other automated setup that you need to do here

data/lib/metacrunch/elasticsearch/client_factory.rb

@@ -1,15 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-
-module Metacrunch::Elasticsearch::ClientFactory
-  def client_factory
-    client_options = {
-      host: @host,
-      hosts: @hosts,
-      url: @url,
-      urls: @urls
-    }.compact
-
-    Elasticsearch::Client.new(client_options)
-  end
-end

data/lib/metacrunch/elasticsearch/index_creator.rb

@@ -1,77 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-require_relative "./client_factory"
-require_relative "./options_helpers"
-
-class Metacrunch::Elasticsearch::IndexCreator
-  include Metacrunch::Elasticsearch::ClientFactory
-  include Metacrunch::Elasticsearch::OptionsHelpers
-
-  attr_accessor :default_mapping
-  attr_accessor :delete_existing_index
-  attr_accessor :logger
-  attr_accessor :settings
-
-  def initialize(options = {})
-    (@client_args = options).deep_symbolize_keys!
-    extract_options!(@client_args, :_client_options_, :default_mapping, :delete_existing_index, :logger, :number_of_shards, :number_of_replicas, :settings)
-    raise ArgumentError.new("You have to supply an index name!") if @client_args[:index].blank?
-  end
-
-  def call(items = [])
-    client = client_factory
-    logger = @logger
-
-    if client.indices.exists?(@client_args)
-      if @delete_existing_index == true
-        client.indices.delete(@client_args)
-        log_index_deleted(logger, @client_args[:index], client) if logger
-      elsif @delete_existing_index == false
-        return
-      else
-        raise Metacrunch::Elasticsearch::IndexAlreadyExistsError
-      end
-    end
-
-    client.indices.create(@client_args.merge(
-      {
-        body: {
-          number_of_shards: @number_of_shards,
-          number_of_replicas: @number_of_replicas,
-          settings: @settings
-        }.compact
-      }
-    ))
-
-    log_index_created(logger, @client_args[:index], client) if logger
-
-    if @default_mapping
-      client.indices.put_mapping(
-        @client_args.merge(
-          type: "_default_",
-          body: {
-            _default_: @default_mapping
-          }
-        )
-      )
-    end
-  end
-
-  private
-
-  def log_index_created(logger, index, client)
-    paths = client.transport.hosts.map do |_host|
-      "#{_host[:host]}:#{_host[:port]}"
-    end
-
-    logger.info("Index #{index} created at #{paths}")
-  end
-
-  def log_index_deleted(logger, index, client)
-    paths = client.transport.hosts.map do |_host|
-      "#{_host[:host]}:#{_host[:port]}"
-    end
-
-    logger.info("Index #{index} deleted at #{paths}")
-  end
-end

data/lib/metacrunch/elasticsearch/indexer.rb

@@ -1,90 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-require_relative "./client_factory"
-require_relative "./options_helpers"
-
-class Metacrunch::Elasticsearch::Indexer
-  include Metacrunch::Elasticsearch::ClientFactory
-  include Metacrunch::Elasticsearch::OptionsHelpers
-
-  attr_accessor :bulk_size
-  attr_accessor :callbacks
-  attr_accessor :id_accessor
-  attr_accessor :index
-  attr_accessor :logger
-  attr_accessor :type
-
-  def initialize(options = {})
-    (@client_args = options).deep_symbolize_keys!
-    extract_options!(@client_args, :_client_options_, :bulk_size, :callbacks, :id_accessor, :index, :logger, :type)
-    raise ArgumentError.new("You have to supply an index name!") if @index.blank?
-  end
-
-  def call(items = [])
-    logger = @logger
-
-    if (slice_size = @bulk_size || items.length) > 0
-      client = client_factory
-
-      items.each_slice(slice_size) do |_item_slice|
-        # bodies is an array to allow slicing in case of HTTP content length exceed
-        bodies = [_item_slice.inject([]) { |_memo, _item| _memo.concat bulk_item_factory(_item) }]
-
-        bulk_responses =
-        begin
-          bodies.map do |_body|
-            client.bulk body: _body
-          end
-        rescue
-          logger.info "Bulk index failed. Decreasing bulk size temporary and trying again." if logger
-
-          bodies = bodies.inject([]) do |_memo, _body|
-            # Since we have to work with the bulk request body instead if the original items
-            # the bodys length has to be a multiple of 2 in any case. .fdiv(2).fdiv(2).ceil * 2
-            # ensures this. Example 3698.fdiv(2).fdiv(2).fdiv(2).ceil * 2 == 1850
-            _memo.concat(_body.each_slice(_body.length.fdiv(2).fdiv(2).ceil * 2).to_a)
-          end
-
-          retry
-        end
-
-        bulk_responses.each do |_bulk_response|
-          log_items_indexed(logger, _bulk_response["items"].length, client) if logger
-
-          if after_indexed_callback = (@callbacks || {})[:after_indexed]
-            _item_slice.zip(_bulk_response["items"]).each do |_item, _item_response|
-              after_indexed_callback.call(_item, _item_response)
-            end
-          end
-        end
-      end
-    end
-  end
-
-  private
-
-  def bulk_item_factory(item)
-    [
-      { index: { _index: @index, _type: @type, _id: id(item) }.compact },
-      item.to_h
-    ]
-  end
-
-  def id(item)
-    if @id_accessor
-      if @id_accessor.respond_to?(:call)
-        @id_accessor.call(item)
-      else
-        item[@id_accessor]
-      end
-    end
-  end
-
-  def log_items_indexed(logger, amount, client)
-    paths = client.transport.hosts.map do |_host|
-      "#{_host[:host]}:#{_host[:port]}/#{@index}/#{@type}"
-    end
-
-    logger.info("Indexed #{amount} items to #{paths}")
-  end
-end

data/lib/metacrunch/elasticsearch/options_helpers.rb

@@ -1,30 +0,0 @@
-require_relative "../elasticsearch"
-
-module Metacrunch::Elasticsearch::OptionsHelpers
-  def extract_options!(options, *keys)
-    keys = keys
-    .map do |_key|
-      _key == :_client_options_ ? [:host, :hosts, :url, :urls] : _key
-    end
-    .flatten
-
-    options
-    .delete_if do |_key, _value|
-      if keys.include?(_key)
-        instance_variable_set("@#{_key}", _value)
-        true # else if _value is falsy, the key does not get deleted
-      end
-    end
-  end
-
-  def normalize_options!(options)
-    {
-      index: options[:index],
-      body: options.select { |_key, _| _key != :index }
-    }
-    .tap(&:compact!)
-    .try do |_result|
-      options.clear.merge!(_result)
-    end
-  end
-end

data/lib/metacrunch/elasticsearch/reader.rb

@@ -1,63 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-
-module Metacrunch
-  module Elasticsearch
-    class Reader
-
-      DEFAULT_SCAN_SIZE          = 250
-      DEFAULT_SCROLL_EXPIRY_TIME = 10.minutes
-
-
-      def initialize(uri, body, log: false)
-        unless uri.starts_with?("elasticsearch://")
-          raise ArgumentError, "URI must be an elasticsearch URI (elasticsearch://...)"
-        end
-
-        @uri  = URI(uri)
-        @body = body
-        @log  = log
-      end
-
-      def each(&block)
-        return enum_for(__method__) unless block_given?
-
-        search_result = client.search({
-          body: @body,
-          index: @uri.index,
-          type: @uri.type,
-          scroll: "#{DEFAULT_SCROLL_EXPIRY_TIME}s",
-          search_type: "scan",
-          size: DEFAULT_SCAN_SIZE
-        })
-
-        while (
-          search_result = client.scroll(
-            scroll: "#{DEFAULT_SCROLL_EXPIRY_TIME}s",
-            scroll_id: search_result["_scroll_id"]
-          ) and # don't use &&, the semantic of 'and' is important here
-          search_result["hits"]["hits"].present?
-        ) do
-          search_result["hits"]["hits"].each do |_hit|
-            yield(_hit)
-          end
-        end
-      end
-
-      def count
-        client.count({
-          body: { query: @body[:query] },
-          index: @uri.index,
-          type: @uri.type
-        })["count"]
-      end
-
-    private
-
-      def client
-        @client ||= ::Elasticsearch::Client.new(host: @uri.host, port: @uri.port, log: @log)
-      end
-
-    end
-  end
-end

data/lib/metacrunch/elasticsearch/searcher.rb

@@ -1,56 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-require_relative "./client_factory"
-require_relative "./options_helpers"
-
-class Metacrunch::Elasticsearch::Searcher
-  include Enumerable
-  include Metacrunch::Elasticsearch::ClientFactory
-  include Metacrunch::Elasticsearch::OptionsHelpers
-
-  DEFAULT_BODY = { query: { match_all: {} } }
-  DEFAULT_SCAN_SIZE = 200 # per shard
-  DEFAULT_SCROLL_EXPIRY_TIME = 10.minutes
-
-  attr_accessor :bulk_size
-  attr_accessor :index
-  attr_accessor :scan_size
-  attr_accessor :scroll_expiry_time
-  attr_accessor :type
-
-  def initialize(options = {})
-    options.deep_symbolize_keys!
-    extract_options!(options, :_client_options_, :bulk_size, :index, :scan_size, :scroll_expiry_time, :type)
-    @body = options.presence || DEFAULT_BODY
-  end
-
-  def call(items = [])
-    @docs_enumerator ||= @bulk_size ? each_slice(@bulk_size) : [each.to_a].to_enum
-    items.concat(@docs_enumerator.next)
-  end
-
-  def each
-    return enum_for(__method__) unless block_given?
-    client = client_factory
-
-    search_result = client.search({
-      body: @body,
-      index: @index,
-      scroll: "#{@scroll_expiry_time || DEFAULT_SCROLL_EXPIRY_TIME}s",
-      search_type: "scan",
-      size: @scan_size || DEFAULT_SCAN_SIZE
-    })
-
-    while (
-      search_result = client.scroll(
-        scroll: "#{DEFAULT_SCROLL_EXPIRY_TIME}s",
-        scroll_id: search_result["_scroll_id"]
-      ) and # don't use &&, the semantic of and is important here
-      search_result["hits"]["hits"].present?
-    ) do
-      search_result["hits"]["hits"].each do |_hit|
-        yield _hit
-      end
-    end
-  end
-end

data/lib/metacrunch/elasticsearch/uri.rb

@@ -1,31 +0,0 @@
-require "uri"
-require_relative "../elasticsearch"
-
-
-module Metacrunch
-  module Elasticsearch
-    class URI < URI::Generic
-
-      DEFAULT_PORT = 9200
-
-      def index
-        splitted_path[0]
-      end
-
-      def type
-        splitted_path[1]
-      end
-
-    private
-
-      def splitted_path
-        path.split("/").map(&:presence).compact
-      end
-
-    end
-  end
-end
-
-module URI
-  @@schemes['ELASTICSEARCH'] = Metacrunch::Elasticsearch::URI
-end

data/lib/metacrunch/elasticsearch/writer.rb

@@ -1,59 +0,0 @@
-require "elasticsearch"
-require_relative "../elasticsearch"
-
-module Metacrunch
-  module Elasticsearch
-    class Writer
-
-      def initialize(uri, log: false, bulk_size: 250, autoflush: true)
-        unless uri.starts_with?("elasticsearch://")
-          raise ArgumentError, "URI must be an elasticsearch URI (elasticsearch://...)"
-        end
-
-        @uri       = URI(uri)
-        @log       = log
-        @bulk_size = bulk_size
-        @buffer    = []
-        @autoflush = autoflush
-      end
-
-      def write(data, options = {})
-        id = data.delete(:id) || data.delete(:_id)
-        raise ArgumentError, "Missing id. You must provide 'id' or '_id' as part of the data" unless id
-
-        @buffer << {
-          _index: @uri.index,
-          _type: @uri.type,
-          _id: id,
-          data: data
-        }
-
-        flush if @autoflush && @bulk_size > 0 && @buffer.length >= @bulk_size
-
-        true
-      end
-
-      def flush
-        if @buffer.length > 0
-          result = client.bulk(body: @buffer.inject([]){ |_body, _data| _body << { index: _data } })
-          raise RuntimeError if result["errors"]
-        end
-
-        true
-      ensure
-        @buffer = []
-      end
-
-      def close
-        flush
-      end
-
-    private
-
-      def client
-        @client ||= ::Elasticsearch::Client.new(host: @uri.host, port: @uri.port, log: @log)
-      end
-
-    end
-  end
-end