elasticsearch-extensions 0.0.3 → 0.0.33

data/elasticsearch-extensions.gemspec

@@ -1,41 +1,63 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # coding: utf-8
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'elasticsearch/extensions/version'
 
 Gem::Specification.new do |s|
-  s.name          = "elasticsearch-extensions"
+  s.name          = 'elasticsearch-extensions'
   s.version       = Elasticsearch::Extensions::VERSION
-  s.authors       = ["Karel Minarik"]
-  s.email         = ["karel.minarik@elasticsearch.org"]
+  s.authors       = ['Karel Minarik']
+  s.email         = ['karel.minarik@elasticsearch.org']
   s.description   = %q{Extensions for the Elasticsearch Rubygem}
   s.summary       = %q{Extensions for the Elasticsearch Rubygem}
-  s.homepage      = ""
-  s.license       = "Apache 2"
-
+  s.homepage      = 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html'
+  s.license       = 'Apache-2.0'
+  s.metadata = {
+    'homepage_uri' => 'https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current/index.html',
+    'changelog_uri' => 'https://github.com/elastic/elasticsearch-ruby/blob/master/CHANGELOG.md',
+    'source_code_uri' => 'https://github.com/elastic/elasticsearch-ruby/tree/master/elasticsearch-extensions',
+    'bug_tracker_uri' => 'https://github.com/elastic/elasticsearch-ruby/issues'
+  }
   s.files         = `git ls-files`.split($/)
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
-  s.require_paths = ["lib"]
-
-  s.add_dependency "elasticsearch", '~> 0.4.0'
-  s.add_dependency "ansi"
-
+  s.require_paths = ['lib']
 
-  s.add_development_dependency "bundler", "> 1"
-  s.add_development_dependency "rake"
+  s.add_dependency 'ansi'
+  s.add_dependency 'elasticsearch'
 
-  s.add_development_dependency "shoulda-context"
-  s.add_development_dependency "mocha"
-  s.add_development_dependency "turn"
-  s.add_development_dependency "yard"
-  s.add_development_dependency "ruby-prof"
-  s.add_development_dependency "ci_reporter"
+  s.add_development_dependency 'ruby-prof' unless defined?(JRUBY_VERSION) || defined?(Rubinius)
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'rake', '~> 12.3'
+  s.add_development_dependency 'awesome_print'
+  s.add_development_dependency 'shoulda-context'
+  s.add_development_dependency 'mocha'
+  s.add_development_dependency 'minitest', '~> 5'
+  s.add_development_dependency 'minitest-reporters', '~> 1'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'yard'
+  s.add_development_dependency 'cane'
+  s.add_development_dependency 'pry'
 
-  if defined?(RUBY_VERSION) && RUBY_VERSION > '1.9'
-    s.add_development_dependency "simplecov"
-    s.add_development_dependency "cane"
-    s.add_development_dependency "require-prof"
-    s.add_development_dependency "coveralls"
+  unless defined?(JRUBY_VERSION)
+    s.add_development_dependency 'oj'
+    s.add_development_dependency 'patron'
   end
 end

data/lib/elasticsearch/extensions/ansi/actions.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 module Elasticsearch
@@ -100,7 +117,7 @@ module Elasticsearch
             output << ""
           end
           output << Helpers.___
-          output << "#{hits.size.to_s.ansi(:bold)} of #{json['hits']['total'].to_s.ansi(:bold)} results".ansi(:faint)
+          output << "#{hits.size.to_s.ansi(:bold)} of #{json['hits']['total']['value'].to_s.ansi(:bold)} results".ansi(:faint)
 
           output.join("\n")
         end
@@ -157,6 +174,33 @@ module Elasticsearch
           output.join("\n")
         end
 
+        # Display histogram facets
+        #
+        def display_histogram_facets(json, options={})
+          return unless json['facets']
+
+          output = [] << ''
+
+          facets = json['facets'].select { |name, values| values['_type'] == 'histogram' }
+          facets.each do |name, values|
+            max     = values['entries'].map { |t| t['count'] }.max
+            padding = 27
+            ratio   = ((Helpers.width)-padding)/max.to_f
+
+            histogram = values['entries']
+            histogram.each_with_index do |segment, i|
+              key = (i == 0) ? "<#{histogram[1]['key']}ms" : "#{segment['key']}ms"
+
+              output << key.rjust(7) +
+                        ' ' +
+                        '█' * (segment['count']*ratio).ceil +
+                        " [#{segment['count']}]"
+            end
+          end
+
+          output.join("\n")
+        end
+
         # Display statistical facets
         #
         def display_statistical_facets(json, options={})

data/lib/elasticsearch/extensions/ansi/helpers.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 module Elasticsearch

data/lib/elasticsearch/extensions/ansi/response.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 module Elasticsearch
   module Extensions
     module ANSI
@@ -13,7 +30,7 @@ module Elasticsearch
         # representation of the Elasticsearch response for:
         #
         # * Search results (hits and highlights)
-        # * Facets (terms, statistical, date_histogram)
+        # * Facets (terms, statistical, histogram, date_histogram)
         # * Analyze API output
         # * Shard allocation
         #
@@ -31,7 +48,11 @@ module Elasticsearch
             Actions.send(m, self, options)
           end
 
-          output.compact.join("\n")
+          unless output.compact.empty?
+            output.compact.join("\n")
+          else
+            self.respond_to?(:awesome_inspect) ? self.awesome_inspect : self.inspect
+          end
         end
       end
 

data/lib/elasticsearch/extensions/ansi.rb

@@ -1,3 +1,20 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 # encoding: utf-8
 
 require 'elasticsearch/extensions'

data/lib/elasticsearch/extensions/backup.rb

@@ -0,0 +1,202 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# encoding: utf-8
+
+require 'pathname'
+require 'fileutils'
+
+require 'multi_json'
+
+begin
+  require 'oj'
+rescue LoadError
+  warn('The "oj" gem could not be loaded. JSON parsing and serialization performance may not be optimal.')
+end
+
+begin
+  require 'patron'
+rescue LoadError
+  warn('The "patron" gem could not be loaded. HTTP requests may not be performed optimally.')
+end
+
+
+require 'elasticsearch'
+
+module Backup
+  module Database
+
+    # Integration with the Backup gem [http://backup.github.io/backup/v4/]
+    #
+    # This extension allows to backup Elasticsearch indices as flat JSON files on the disk.
+    #
+    # @example Use the Backup gem's DSL to configure the backup
+    #
+    #     require 'elasticsearch/extensions/backup'
+    #
+    #     Model.new(:elasticsearch_backup, 'Elasticsearch') do
+    #
+    #       database Elasticsearch do |db|
+    #         db.url     = 'http://localhost:9200'
+    #         db.indices = 'articles,people'
+    #         db.size    = 500
+    #         db.scroll  = '10m'
+    #       end
+    #
+    #       store_with Local do |local|
+    #         local.path = '/tmp/backups'
+    #         local.keep = 3
+    #       end
+    #
+    #       compress_with Gzip
+    #     end
+    #
+    # Perform the backup with the Backup gem's command line utility:
+    #
+    #     $ backup perform -t elasticsearch_backup
+    #
+    # The Backup gem can store your backup files on S3, Dropbox and other
+    # cloud providers, send notifications about the operation, and so on;
+    # read more in the gem documentation.
+    #
+    # @example Use the integration as a standalone script (eg. in a Rake task)
+    #
+    #     require 'backup'
+    #     require 'elasticsearch/extensions/backup'
+    #
+    #     Backup::Logger.configure do
+    #       logfile.enabled   = true
+    #       logfile.log_path  = '/tmp/backups/log'
+    #     end; Backup::Logger.start!
+    #
+    #     backup  = Backup::Model.new(:elasticsearch, 'Backup Elasticsearch') do
+    #       database Backup::Database::Elasticsearch do |db|
+    #         db.indices = 'test'
+    #       end
+    #
+    #       store_with Backup::Storage::Local do |local|
+    #         local.path = '/tmp/backups'
+    #       end
+    #     end
+    #
+    #     backup.perform!
+    #
+    # @example A simple recover script for the backup created in the previous examples
+    #
+    #     PATH = '/path/to/backup/'
+    #
+    #     require 'elasticsearch'
+    #     client  = Elasticsearch::Client.new log: true
+    #     payload = []
+    #
+    #     Dir[ File.join( PATH, '**', '*.json' ) ].each do |file|
+    #       document = MultiJson.load(File.read(file))
+    #       item = document.merge(data: document['_source'])
+    #       document.delete('_source')
+    #       document.delete('_score')
+    #
+    #       payload << { index: item }
+    #
+    #       if payload.size == 100
+    #         client.bulk body: payload
+    #         payload = []
+    #       end
+    #
+    #       client.bulk body: payload
+    #     end
+    #
+    # @see http://backup.github.io/backup/v4/
+    #
+    class Elasticsearch < Base
+      class Error < ::Backup::Error; end
+
+      attr_accessor :url,
+                    :indices,
+                    :size,
+                    :scroll
+
+      attr_accessor :mode
+
+      def initialize(model, database_id = nil, &block)
+        super
+
+        @url     ||= 'http://localhost:9200'
+        @indices ||= '_all'
+        @size    ||= 100
+        @scroll  ||= '10m'
+        @mode    ||= 'single'
+
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        super
+
+        case mode
+          when 'single'
+            __perform_single
+          else
+            raise Error, "Unsupported mode [#{mode}]"
+        end
+
+        log!(:finished)
+      end
+
+      def client
+        @client ||= ::Elasticsearch::Client.new url: url, logger: logger
+      end
+
+      def path
+        Pathname.new File.join(dump_path , dump_filename.downcase)
+      end
+
+      def logger
+        logger = Backup::Logger.__send__(:logger)
+        logger.instance_eval do
+          def debug(*args);end
+          # alias :debug :info
+          alias :fatal :warn
+        end
+        logger
+      end
+
+
+      def __perform_single
+        r = client.search index: indices, search_type: 'scan', scroll: scroll, size: size
+        raise Error, "No scroll_id returned in response:\n#{r.inspect}" unless r['_scroll_id']
+
+        while r = client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) and not r['hits']['hits'].empty? do
+          r['hits']['hits'].each do |hit|
+            FileUtils.mkdir_p "#{path.join hit['_index'], hit['_type']}"
+            File.open("#{path.join hit['_index'], hit['_type'], __sanitize_filename(hit['_id'])}.json", 'w') do |file|
+              file.write MultiJson.dump(hit)
+            end
+          end
+        end
+      end
+
+      def __sanitize_filename name
+        name
+          .encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "�")
+          .strip
+          .tr("\u{202E}%$|:;/\t\r\n\\", "-")
+      end
+    end
+  end
+end
+
+::Backup::Config::DSL::Elasticsearch = ::Backup::Database::Elasticsearch

data/lib/elasticsearch/extensions/reindex.rb

@@ -0,0 +1,187 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# encoding: utf-8
+
+module Elasticsearch
+  module Extensions
+
+    # This module allows copying documents from one index/cluster to another one
+    #
+    # When required together with the client, it will add the `reindex` method
+    #
+    # @see Reindex::Reindex.initialize
+    # @see Reindex::Reindex#perform
+    #
+    # @see http://www.rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions#reindex-instance_method
+    #
+    module Reindex
+
+      # Initialize a new instance of the Reindex class (shortcut)
+      #
+      # @see Reindex::Reindex.initialize
+      #
+      def new(arguments={})
+        Reindex.new(arguments)
+      end; extend self
+
+      module API
+        # Copy documents from one index into another and refresh the destination index
+        #
+        # @example
+        #     client.reindex source: { index: 'test1' }, dest: { index: 'test2' }, refresh: true
+        #
+        # The method allows all the options as {Reindex::Reindex.new}.
+        #
+        # This method will be mixed into the Elasticsearch client's API, if available.
+        #
+        def reindex(arguments={})
+          arguments[:source] ||= {}
+          arguments[:source][:client] = self
+          Reindex.new(arguments).perform
+        end
+      end
+
+      # Include the `reindex` method in the API and client, if available
+      Elasticsearch::API::Actions.__send__ :include, API if defined?(Elasticsearch::API::Actions)
+      Elasticsearch::Transport::Client.__send__ :include, API if defined?(Elasticsearch::Transport::Client) && defined?(Elasticsearch::API)
+
+      # Copy documents from one index into another
+      #
+      # @example Copy documents to another index
+      #
+      #   client  = Elasticsearch::Client.new
+      #   reindex = Elasticsearch::Extensions::Reindex.new \
+      #               source: { index: 'test1', client: client },
+      #               dest: { index: 'test2' }
+      #
+      #   reindex.perform
+      #
+      # @example Copy documents to a different cluster
+      #
+      #     source_client  = Elasticsearch::Client.new url: 'http://localhost:9200'
+      #     destination_client  = Elasticsearch::Client.new url: 'http://localhost:9250'
+      #
+      #     reindex = Elasticsearch::Extensions::Reindex.new \
+      #                 source: { index: 'test', client: source_client },
+      #                 dest: { index: 'test', client: destination_client }
+      #     reindex.perform
+      #
+      # @example Transform the documents during re-indexing
+      #
+      #     reindex = Elasticsearch::Extensions::Reindex.new \
+      #                 source: { index: 'test1', client: client },
+      #                 dest: { index: 'test2' },
+      #                 transform: lambda { |doc| doc['_source']['category'].upcase! }
+      #
+      #
+      # The reindexing process works by "scrolling" an index and sending
+      # batches via the "Bulk" API to the destination index/cluster
+      #
+      # @option arguments [String] :source The source index/cluster definition (*Required*)
+      # @option arguments [String] :dest The destination index/cluster definition (*Required*)
+      # @option arguments [Proc] :transform A block which will be executed for each document
+      # @option arguments [Integer] :batch_size The size of the batch for scroll operation (Default: 1000)
+      # @option arguments [String] :scroll The timeout for the scroll operation (Default: 5min)
+      # @option arguments [Boolean] :refresh Whether to refresh the destination index after
+      #                                      the operation is completed (Default: false)
+      #
+      # Be aware, that if you want to change the destination index settings and/or mappings,
+      # you have to do so in advance by using the "Indices Create" API.
+      #
+      # Note, that there is a native "Reindex" API in Elasticsearch 2.3.x and higer versions,
+      # which will be more performant than the Ruby version.
+      #
+      # @see http://www.rubydoc.info/gems/elasticsearch-api/Elasticsearch/API/Actions#reindex-instance_method
+      #
+      class Reindex
+        attr_reader :arguments
+
+        def initialize(arguments={})
+          [
+            [:source, :index],
+            [:source, :client],
+            [:dest, :index]
+          ].each do |required_option|
+            value = required_option.reduce(arguments) { |sum, o| sum = sum[o] ? sum[o] : {}  }
+
+            raise ArgumentError,
+                  "Required argument '#{Hash[*required_option]}' missing" if \
+                  value.respond_to?(:empty?) ? value.empty? : value.nil?
+          end
+
+          @arguments = {
+            batch_size: 1000,
+            scroll: '5m',
+            refresh: false
+          }.merge(arguments)
+
+          arguments[:dest][:client] ||= arguments[:source][:client]
+        end
+
+        # Performs the operation
+        #
+        # @return [Hash] A Hash with the information about the operation outcome
+        #
+        def perform
+          output = { errors: 0 }
+
+          response = arguments[:source][:client].search(
+            index: arguments[:source][:index],
+            scroll: arguments[:scroll],
+            size: arguments[:batch_size]
+          )
+
+          documents = response['hits']['hits']
+
+          unless documents.empty?
+            bulk_response = __store_batch(documents)
+            output[:errors] += bulk_response['items'].select { |k, v| k.values.first['error'] }.size
+          end
+
+          while response = arguments[:source][:client].scroll(scroll_id: response['_scroll_id'], scroll: arguments[:scroll]) do
+            documents = response['hits']['hits']
+            break if documents.empty?
+
+            bulk_response = __store_batch(documents)
+            output[:errors] += bulk_response['items'].select { |k, v| k.values.first['error'] }.size
+          end
+
+          arguments[:dest][:client].indices.refresh index: arguments[:dest][:index] if arguments[:refresh]
+
+          output
+        end
+
+        def __store_batch(documents)
+          body = documents.map do |doc|
+            doc['_index'] = arguments[:dest][:index]
+
+            arguments[:transform].call(doc) if arguments[:transform]
+
+            doc['data'] = doc['_source']
+            doc.delete('_score')
+            doc.delete('_source')
+
+            { index: doc }
+          end
+
+          arguments[:dest][:client].bulk body: body
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/extensions/test/cluster/tasks.rb

@@ -0,0 +1,30 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'elasticsearch/extensions/test/cluster'
+
+namespace :elasticsearch do
+  desc "Start Elasticsearch cluster for tests"
+  task :start do
+    Elasticsearch::Extensions::Test::Cluster.start
+  end
+
+  desc "Stop Elasticsearch cluster for tests"
+  task :stop do
+    Elasticsearch::Extensions::Test::Cluster.stop
+  end
+end