metacrunch-elasticsearch 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a74d43ab5c9f961f3efd802650e78022fb51e1e8
-  data.tar.gz: 047c7af302f42d06dc24afa823fa46d2fa67a30a
+  metadata.gz: 17ba1fec96c7af64b5cdf8283dfee2fe0b4eadf1
+  data.tar.gz: fb673a9a460c7d7f82c96dfedc6a48fd26fdc971
 SHA512:
-  metadata.gz: ccbddc5b0846556dbbb7409e1b718e716316f56fd8b1dc46c788c10f11fe7406f46db2d1788360bb7bed5703d94885da15e7ad380da17961b2f508c37a34f84b
-  data.tar.gz: cb7e01749ab51e45702bfced9967860376bc41ee35ef40bf78f692f8d5df57fb88eb16e954e31d6251650df9ca0a96e3fcd6b1822190e14bbbc2b34b6a7d3157
+  metadata.gz: 106cdda72d6b43bf2d52392b1f3d45e68bbd679427ec1ca582a265dc904d286415ce8e35a0464e867eca550661ef65232e983e72812c4bd815dbbb5c050dcf34
+  data.tar.gz: 7d3552381f7dbf8fab91d2d1e8a2ca8cc964304bfe2105baa760eeba71cbadd12748ffe6ef03b2a425f343fa37caf51dace45d2534eec8b783f4d30516ee658a

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "2.0"
+  - "2.1"
+  - "2.2"

data/Gemfile

@@ -1,20 +1,28 @@
 source "https://rubygems.org"
 
+# Specify your gem's dependencies in your gemspec
 gemspec
 
-gem "metacrunch", ">= 2.1.0", github: "ubpb/metacrunch", branch: "master"
+group :development do
+  gem "bundler",         ">= 1.10"
+  gem "rake"
+  gem "rspec",           ">= 3.0.0",  "< 4.0.0"
+  gem "simplecov",       ">= 0.8.0"
+  gem "vcr",             ">= 2.9.0",  "< 3.0.0"
+  gem "webmock",         ">= 1.19.0", "< 2.0.0"
 
-gem "rake"
-gem "rspec", "~> 3.2.0"
-
-if !ENV["CI"]
-  group :development do
+  if !ENV["CI"]
     gem "hashdiff"
     gem "pry",                "~> 0.9.12.6"
     gem "pry-byebug",         "<= 1.3.2"
-    gem "pry-rescue",         "~> 1.4.1", github: "ConradIrwin/pry-rescue", branch: :master
+    gem "pry-rescue",         "~> 1.4.2"
     gem "pry-stack_explorer", "~> 0.4.9.1"
     gem "pry-syntax-hacks",   "~> 0.0.6"
   end
 end
 
+group :test do
+  gem "codeclimate-test-reporter", require: nil
+end
+
+gem "metacrunch", github: "ubpb/metacrunch", branch: :master

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "metascrunch/elasticsearch"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+require "pry"
+Pry.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/metacrunch/elasticsearch/client_factory.rb

@@ -0,0 +1,15 @@
+require "elasticsearch"
+require_relative "../elasticsearch"
+
+module Metacrunch::Elasticsearch::ClientFactory
+  def client_factory
+    client_options = {
+      host: @host,
+      hosts: @hosts,
+      url: @url,
+      urls: @urls
+    }.compact
+
+    Elasticsearch::Client.new(client_options)
+  end
+end

data/lib/metacrunch/elasticsearch/index_creator.rb

@@ -0,0 +1,76 @@
+require "elasticsearch"
+require "metacrunch/processor"
+require_relative "../elasticsearch"
+require_relative "./client_factory"
+require_relative "./options_helpers"
+
+class Metacrunch::Elasticsearch::IndexCreator < Metacrunch::Processor
+  include Metacrunch::Elasticsearch::ClientFactory
+  include Metacrunch::Elasticsearch::OptionsHelpers
+
+  attr_accessor :default_mapping
+  attr_accessor :delete_existing_index
+  attr_accessor :logger
+
+  def initialize(options = {})
+    (@client_args = options).deep_symbolize_keys!
+    extract_options!(@client_args, :_client_options_, :default_mapping, :delete_existing_index, :logger, :number_of_shards, :number_of_replicas)
+    raise ArgumentError.new("You have to supply an index name!") if @client_args[:index].blank?
+  end
+
+  def call(items = [], pipeline = nil)
+    client = client_factory
+    logger = pipeline.try(:logger) || @logger
+
+    if client.indices.exists?(@client_args)
+      if @delete_existing_index == true
+        client.indices.delete(@client_args)
+        log_index_deleted(logger, @client_args[:index], client) if logger
+      elsif @delete_existing_index == false
+        return
+      else
+        raise Metacrunch::Elasticsearch::IndexAlreadyExistsError
+      end
+    end
+
+    client.indices.create(@client_args.merge(
+      {
+        body: {
+          number_of_shards: @number_of_shards,
+          number_of_replicas: @number_of_replicas
+        }.compact
+      }
+    ))
+
+    log_index_created(logger, @client_args[:index], client) if logger
+
+    if @default_mapping
+      client.indices.put_mapping(
+        @client_args.merge(
+          type: "_default_",
+          body: {
+            _default_: @default_mapping
+          }
+        )
+      )
+    end
+  end
+
+  private
+
+  def log_index_created(logger, index, client)
+    paths = client.transport.hosts.map do |_host|
+      "#{_host[:host]}:#{_host[:port]}"
+    end
+
+    logger.info("Index #{index} created at #{paths}")
+  end
+
+  def log_index_deleted(logger, index, client)
+    paths = client.transport.hosts.map do |_host|
+      "#{_host[:host]}:#{_host[:port]}"
+    end
+
+    logger.info("Index #{index} deleted at #{paths}")
+  end
+end

data/lib/metacrunch/elasticsearch/indexer.rb

@@ -0,0 +1,91 @@
+require "elasticsearch"
+require "metacrunch/processor"
+require_relative "../elasticsearch"
+require_relative "./client_factory"
+require_relative "./options_helpers"
+
+class Metacrunch::Elasticsearch::Indexer < Metacrunch::Processor
+  include Metacrunch::Elasticsearch::ClientFactory
+  include Metacrunch::Elasticsearch::OptionsHelpers
+
+  attr_accessor :bulk_size
+  attr_accessor :callbacks
+  attr_accessor :id_accessor
+  attr_accessor :index
+  attr_accessor :logger
+  attr_accessor :type
+  
+  def initialize(options = {})
+    (@client_args = options).deep_symbolize_keys!
+    extract_options!(@client_args, :_client_options_, :bulk_size, :callbacks, :id_accessor, :index, :logger, :type)
+    raise ArgumentError.new("You have to supply an index name!") if @index.blank?
+  end
+
+  def call(items = [], pipeline = nil)
+    logger = pipeline.try(:logger) || @logger
+
+    if (slice_size = @bulk_size || items.length) > 0
+      client = client_factory
+
+      items.each_slice(slice_size) do |_item_slice|
+        # bodies is an array to allow slicing in case of HTTP content length exceed
+        bodies = [_item_slice.inject([]) { |_memo, _item| _memo.concat bulk_item_factory(_item) }]
+
+        bulk_responses =
+        begin
+          bodies.map do |_body|
+            client.bulk body: _body
+          end
+        rescue
+          logger.info "Bulk index failed. Decreasing bulk size temporary and trying again." if logger
+
+          bodies = bodies.inject([]) do |_memo, _body|
+            # Since we have to work with the bulk request body instead if the original items
+            # the bodys length has to be a multiple of 2 in any case. .fdiv(2).fdiv(2).ceil * 2
+            # ensures this. Example 3698.fdiv(2).fdiv(2).fdiv(2).ceil * 2 == 1850
+            _memo.concat(_body.each_slice(_body.length.fdiv(2).fdiv(2).ceil * 2).to_a)
+          end
+
+          retry
+        end
+
+        bulk_responses.each do |_bulk_response|
+          log_items_indexed(logger, _bulk_response["items"].length, client) if logger
+
+          if after_indexed_callback = (@callbacks || {})[:after_indexed]
+            _item_slice.zip(_bulk_response["items"]).each do |_item, _item_response|
+              after_indexed_callback.call(_item, _item_response)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  def bulk_item_factory(item)
+    [
+      { index: { _index: @index, _type: @type, _id: id(item) }.compact },
+      item.to_h
+    ]
+  end
+
+  def id(item)
+    if @id_accessor
+      if @id_accessor.respond_to?(:call)
+        @id_accessor.call(item)
+      else
+        item[@id_accessor]
+      end
+    end
+  end
+
+  def log_items_indexed(logger, amount, client)
+    paths = client.transport.hosts.map do |_host|
+      "#{_host[:host]}:#{_host[:port]}/#{@index}/#{@type}"
+    end
+
+    logger.info("Indexed #{amount} items to #{paths}")
+  end
+end

data/lib/metacrunch/elasticsearch/options_helpers.rb

@@ -0,0 +1,30 @@
+require_relative "../elasticsearch"
+
+module Metacrunch::Elasticsearch::OptionsHelpers
+  def extract_options!(options, *keys)
+    keys = keys
+    .map do |_key|
+      _key == :_client_options_ ? [:host, :hosts, :url, :urls] : _key
+    end
+    .flatten
+
+    options
+    .delete_if do |_key, _value|
+      if keys.include?(_key)
+        instance_variable_set("@#{_key}", _value)
+        true # else if _value is falsy, the key does not get deleted
+      end
+    end
+  end
+
+  def normalize_options!(options)
+    {
+      index: options[:index],
+      body: options.select { |_key, _| _key != :index }
+    }
+    .tap(&:compact!)
+    .try do |_result|
+      options.clear.merge!(_result)
+    end
+  end
+end

data/lib/metacrunch/elasticsearch/searcher.rb

@@ -0,0 +1,62 @@
+require "elasticsearch"
+require "metacrunch/processor"
+require_relative "../elasticsearch"
+require_relative "./client_factory"
+require_relative "./options_helpers"
+
+class Metacrunch::Elasticsearch::Searcher < Metacrunch::Processor
+  include Enumerable
+  include Metacrunch::Elasticsearch::ClientFactory
+  include Metacrunch::Elasticsearch::OptionsHelpers
+
+  DEFAULT_BODY = { query: { match_all: {} } }
+  DEFAULT_SCAN_SIZE = 200 # per shard
+  DEFAULT_SCROLL_EXPIRY_TIME = 10.minutes
+
+  attr_accessor :bulk_size
+  attr_accessor :index
+  attr_accessor :scan_size
+  attr_accessor :scroll_expiry_time
+  attr_accessor :type
+  
+  def initialize(options = {})
+    options.deep_symbolize_keys!
+    extract_options!(options, :_client_options_, :bulk_size, :index, :scan_size, :scroll_expiry_time, :type)
+    @body = options.presence || DEFAULT_BODY
+  end
+
+  def call(items = [], pipeline = nil)
+    @docs_enumerator ||= @bulk_size ? each_slice(@bulk_size) : [each.to_a].to_enum
+
+    begin
+      items.concat(@docs_enumerator.next)
+    rescue StopIteration
+      pipeline.terminate!
+    end
+  end
+
+  def each
+    return enum_for(__method__) unless block_given?
+    client = client_factory
+
+    search_result = client.search({
+      body: @body,
+      index: @index,
+      scroll: "#{@scroll_expiry_time || DEFAULT_SCROLL_EXPIRY_TIME}s",
+      search_type: "scan",
+      size: @scan_size || DEFAULT_SCAN_SIZE
+    })
+
+    while (
+      search_result = client.scroll(
+        scroll: "#{DEFAULT_SCROLL_EXPIRY_TIME}s",
+        scroll_id: search_result["_scroll_id"]
+      ) and # don't use &&, the semantic of and is important here
+      search_result["hits"]["hits"].present?
+    ) do
+      search_result["hits"]["hits"].each do |_hit|
+        yield _hit
+      end
+    end
+  end
+end

data/lib/metacrunch/elasticsearch/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module Elasticsearch
-    VERSION = "2.0.1"
+    VERSION = "2.1.0"
   end
 end

data/lib/metacrunch/elasticsearch.rb

@@ -3,8 +3,16 @@ require "elasticsearch"
 
 module Metacrunch
   module Elasticsearch
-    require_relative "./elasticsearch/uri"
+    require_relative "./elasticsearch/index_creator"
+    require_relative "./elasticsearch/indexer"
     require_relative "./elasticsearch/reader"
+    require_relative "./elasticsearch/searcher"
+    require_relative "./elasticsearch/uri"
     require_relative "./elasticsearch/writer"
+
+    #
+    # error class are inline to not clutter source files unnecessarily
+    #
+    class IndexAlreadyExistsError < StandardError; end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - René Sprotte
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-09-25 00:00:00.000000000 Z
+date: 2015-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -60,12 +60,21 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
+- ".travis.yml"
 - Gemfile
 - License.txt
 - Rakefile
 - Readme.md
+- bin/console
+- bin/setup
 - lib/metacrunch/elasticsearch.rb
+- lib/metacrunch/elasticsearch/client_factory.rb
+- lib/metacrunch/elasticsearch/index_creator.rb
+- lib/metacrunch/elasticsearch/indexer.rb
+- lib/metacrunch/elasticsearch/options_helpers.rb
 - lib/metacrunch/elasticsearch/reader.rb
+- lib/metacrunch/elasticsearch/searcher.rb
 - lib/metacrunch/elasticsearch/uri.rb
 - lib/metacrunch/elasticsearch/version.rb
 - lib/metacrunch/elasticsearch/writer.rb
@@ -96,4 +105,3 @@ signing_key:
 specification_version: 4
 summary: Metacrunch elasticsearch package
 test_files: []
-has_rdoc: