elasticsearch-extensions 0.0.3 → 0.0.33

data/lib/elasticsearch/extensions/test/profiling.rb

@@ -0,0 +1,124 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'ruby-prof'
+require 'benchmark'
+require 'ansi'
+
+module Elasticsearch
+  module Extensions
+    module Test
+
+      # Allows to define and execute profiling tests within [Shoulda](https://github.com/thoughtbot/shoulda) contexts.
+      #
+      # Measures operations and reports statistics, including code profile.
+      #
+      # Uses the "benchmark" standard library and the "ruby-prof" gem.
+      #
+      #     File: profiling_test.rb
+      #
+      #     require 'test/unit'
+      #     require 'shoulda/context'
+      #     require 'elasticsearch/extensions/test/profiling'
+      #
+      #     class ProfilingTest < Test::Unit::TestCase
+      #       extend Elasticsearch::Extensions::Test::Profiling
+      #
+      #       context "Mathematics" do
+      #         measure "divide numbers", count: 10_000 do
+      #           assert_nothing_raised { 1/2 }
+      #         end
+      #       end
+      #
+      #     end
+      #
+      #     $ QUIET=y ruby profiling_test.rb
+      #
+      #     ...
+      #     ProfilingTest
+      #
+      #     -------------------------------------------------------------------------------
+      #     Context: Mathematics should divide numbers (10000x)
+      #     mean: 0.03ms | avg: 0.03ms | max: 0.14ms
+      #     -------------------------------------------------------------------------------
+      #          PASS (0:00:00.490) test: Mathematics should divide numbers (10000x).
+      #     ...
+      #
+      module Profiling
+
+        # Profiles the passed block of code.
+        #
+        #     measure "divide numbers", count: 10_000 do
+        #      assert_nothing_raised { 1/2 }
+        #     end
+        #
+        # @todo Try to make progress bar not to interfere with tests
+        #
+        def measure(name, options={}, &block)
+          ___          = '-'*ANSI::Terminal.terminal_width
+          test_name    = name
+          suite_name   = self.name.split('::').last
+          context_name = self.context(nil) {}.first.parent.name
+          count        = Integer(ENV['COUNT'] || options[:count] || 1_000)
+          ticks        = []
+          progress     = ANSI::Progressbar.new(suite_name, count) unless ENV['QUIET'] || options[:quiet]
+
+          should "#{test_name} (#{count}x)" do
+            RubyProf.start
+
+            begin
+              count.times do
+                ticks << Benchmark.realtime { self.instance_eval(&block) }
+
+                if progress
+                  RubyProf.pause
+                  progress.inc
+                  RubyProf.resume
+                end
+              end
+            ensure
+              result = RubyProf.stop
+              progress.finish if progress
+            end
+
+            total = result.threads.reduce(0) { |t,info| t += info.total_time; t }
+            mean  = (ticks.sort[(ticks.size/2).round-1])*1000
+            avg   = (ticks.inject {|sum,el| sum += el; sum}.to_f/ticks.size)*1000
+            min   = ticks.min*1000
+            max   = ticks.max*1000
+
+
+            result.eliminate_methods!([/Integer#times|Benchmark.realtime|ANSI::Code#.*|ANSI::ProgressBar#.*/])
+            printer = RubyProf::FlatPrinter.new(result)
+            # printer = RubyProf::GraphPrinter.new(result)
+
+            puts "\n",
+                 ___,
+                 "#{suite_name}: " + ANSI.bold(context_name) + ' should ' + ANSI.bold(name) + " (#{count}x)",
+                 "total: #{sprintf('%.2f', total)}s | " +
+                 "mean: #{sprintf('%.2f', mean)}ms | " +
+                 "avg: #{sprintf('%.2f',  avg)}ms | " +
+                 "min: #{sprintf('%.2f',  min)}ms | " +
+                 "max: #{sprintf('%.2f',  max)}ms",
+                 ___
+            printer.print(STDOUT, {}) unless ENV['QUIET'] || options[:quiet]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/extensions/test/startup_shutdown.rb

@@ -0,0 +1,71 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Elasticsearch
+  module Extensions
+    module Test
+      # Startup/shutdown support for test suites
+      #
+      # Example:
+      #
+      #     class MyTest < Test::Unit::TestCase
+      #       extend Elasticsearch::Extensions::Test::StartupShutdown
+      #
+      #       startup  { puts "Suite starting up..." }
+      #       shutdown { puts "Suite shutting down..." }
+      #     end
+      #
+      # *** IMPORTANT NOTE: **********************************************************
+      #
+      # You have to register the handler for shutdown before requiring 'test/unit':
+      #
+      #     # File: test_helper.rb
+      #     at_exit { MyTest.__run_at_exit_hooks }
+      #     require 'test/unit'
+      #
+      # The API follows Test::Unit 2.0
+      # <https://github.com/test-unit/test-unit/blob/master/lib/test/unit/testcase.rb>
+      #
+      module StartupShutdown
+        @@started           = false
+        @@shutdown_blocks ||= []
+
+        def startup &block
+          return if started?
+          @@started = true
+          yield block if block_given?
+        end
+
+        def shutdown &block
+          @@shutdown_blocks << block if block_given?
+        end
+
+        def started?
+          !! @@started
+        end
+
+        def __run_at_exit_hooks
+          return unless started?
+          STDERR.puts ANSI.faint("Running at_exit hooks...")
+          puts ANSI.faint('-'*80)
+          @@shutdown_blocks.each { |b| b.call }
+          puts ANSI.faint('-'*80)
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/extensions/version.rb

@@ -1,5 +1,22 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 module Elasticsearch
   module Extensions
-    VERSION = "0.0.3"
+    VERSION = '0.0.33'.freeze
   end
 end

data/lib/elasticsearch/extensions.rb

@@ -1,3 +1,22 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# encoding: utf-8
+
 require 'elasticsearch'
 require 'elasticsearch/extensions/version'
 

data/lib/elasticsearch-extensions.rb

@@ -1 +1,6 @@
+# Licensed to Elasticsearch B.V under one or more agreements.
+# Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+# See the LICENSE file in the project root for more information
+#
+
 require 'elasticsearch/extensions'

data/test/ansi/unit/ansi_test.rb

@@ -0,0 +1,66 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'test_helper'
+require 'elasticsearch/extensions/ansi'
+
+class Elasticsearch::Extensions::AnsiTest < Elasticsearch::Test::UnitTestCase
+  context "The ANSI extension" do
+    setup do
+      @client = Elasticsearch::Client.new
+      @client.stubs(:perform_request).returns \
+        Elasticsearch::Transport::Transport::Response.new(200, { "ok" => true, "status" => 200, "name" => "Hit-Maker",
+                                                                 "version" => { "number"     => "0.90.7",
+                                                                                "build_hash" => "abc123",
+                                                                                "build_timestamp"=>"2013-11-13T12:06:54Z", "build_snapshot"=>false, "lucene_version"=>"4.5.1" },
+                                                                 "tagline"=>"You Know, for Search" })
+    end
+
+    should "wrap the response" do
+      response = @client.info
+
+      assert_instance_of Elasticsearch::Extensions::ANSI::ResponseBody, response
+      assert_instance_of Hash, response.to_hash
+    end
+
+    should "extend the response object with `to_ansi`" do
+      response = @client.info
+
+      assert_respond_to response, :to_ansi
+      assert_instance_of String, response.to_ansi
+    end
+
+    should "call the 'awesome_inspect' method when available and no handler found" do
+      @client.stubs(:perform_request).returns \
+        Elasticsearch::Transport::Transport::Response.new(200, {"index-1"=>{"aliases"=>{}}})
+      response = @client.cat.aliases
+
+      response.instance_eval do
+        def awesome_inspect; "---PRETTY---"; end
+      end
+      assert_equal '---PRETTY---', response.to_ansi
+    end
+
+    should "call `to_s` method when no pretty printer or handler found" do
+      @client.stubs(:perform_request).returns \
+        Elasticsearch::Transport::Transport::Response.new(200, {"index-1"=>{"aliases"=>{}}})
+      response = @client.cat.aliases
+
+      assert_equal '{"index-1"=>{"aliases"=>{}}}', response.to_ansi
+    end
+  end
+end

data/test/backup/unit/backup_test.rb

@@ -0,0 +1,131 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'test_helper'
+require 'logger'
+
+# Mock the Backup modules and classes so we're not depending on the gem in the unit test
+#
+module Backup
+  class Error < StandardError; end
+
+  class Logger < ::Logger
+    def self.logger
+      self.new($stderr)
+    end
+  end
+
+  module Config
+    module DSL
+    end
+  end
+
+  module Database
+    class Base
+      def initialize(model, database_id = nil)
+      end
+
+      def dump_path;     'dump_path'; end
+      def dump_filename; 'dump_filename'; end
+
+      def log!(*args)
+        puts "LOGGING..." if ENV['DEBUG']
+      end
+
+      def perform!
+        puts "PERFORMING..." if ENV['DEBUG']
+      end
+    end
+  end
+end
+
+require 'elasticsearch/extensions/backup'
+
+class Elasticsearch::Extensions::BackupTest < Elasticsearch::Test::UnitTestCase
+  context "The Backup gem extension" do
+    setup do
+      @model = stub trigger: true
+      @subject = ::Backup::Database::Elasticsearch.new(@model)
+    end
+
+    should "have a client" do
+      assert_instance_of Elasticsearch::Transport::Client, @subject.client
+    end
+
+    should "have a path" do
+      assert_instance_of Pathname, @subject.path
+    end
+
+    should "have defaults" do
+      assert_equal 'http://localhost:9200', @subject.url
+      assert_equal '_all', @subject.indices
+    end
+
+    should "be configurable" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) do |db|
+        db.url = 'https://example.com'
+        db.indices = 'foo,bar'
+      end
+
+      assert_equal 'https://example.com', @subject.url
+      assert_equal 'foo,bar', @subject.indices
+
+      assert_equal 'example.com', @subject.client.transport.connections.first.host[:host]
+    end
+
+    should "perform the backup" do
+      @subject.expects(:__perform_single)
+      @subject.perform!
+    end
+
+    should "raise an expection for an unsupported type of backup" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) { |db| db.mode = 'foobar' }
+      assert_raise ::Backup::Database::Elasticsearch::Error do
+        @subject.perform!
+      end
+    end
+
+    should "scan and scroll the index" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) { |db| db.indices = 'test' }
+
+      @subject.client
+        .expects(:search)
+        .with do |params|
+          assert_equal 'test', params[:index]
+          true # Thanks, Ruby 2.2
+        end
+        .returns({"_scroll_id" => "abc123"})
+
+      @subject.client
+        .expects(:scroll)
+        .twice
+        .returns({"_scroll_id" => "def456",
+                  "hits" => { "hits" => [ {"_index"=>"test", "_type"=>"doc", "_id"=>"1", "_source"=>{"title"=>"Test"}} ] }
+                })
+        .then
+        .returns({"_scroll_id" => "ghi789",
+                  "hits" => { "hits" => [] }
+                })
+
+      @subject.__perform_single
+    end
+
+    should "sanitize filename" do
+      assert_equal "foo-bar-baz", @subject.__sanitize_filename("foo/bar\nbaz")
+    end
+  end
+end

data/test/reindex/integration/reindex_test.rb

@@ -0,0 +1,107 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'test_helper'
+require 'elasticsearch/extensions/reindex'
+
+class Elasticsearch::Extensions::ReindexIntegrationTest < Elasticsearch::Test::IntegrationTestCase
+  context "The Reindex extension" do
+    setup do
+      @port = (ENV['TEST_CLUSTER_PORT'] || 9250).to_i
+
+      @logger =  ::Logger.new(STDERR)
+      @logger.formatter = proc do |severity, datetime, progname, msg|
+        color = case severity
+          when /INFO/ then :green
+          when /ERROR|WARN|FATAL/ then :red
+          when /DEBUG/ then :cyan
+          else :white
+        end
+        ANSI.ansi(severity[0] + ' ', color, :faint) + ANSI.ansi(msg, :white, :faint) + "\n"
+      end
+
+      @client = Elasticsearch::Client.new host: "#{TEST_HOST}:#{TEST_PORT}", logger: @logger
+      @client.indices.delete index: '_all'
+
+      @client.index index: 'test1', type: 'd', id: 1, body: { title: 'TEST 1', category: 'one' }
+      @client.index index: 'test1', type: 'd', id: 2, body: { title: 'TEST 2', category: 'two' }
+      @client.index index: 'test1', type: 'd', id: 3, body: { title: 'TEST 3', category: 'three' }
+      @client.indices.refresh index: 'test1'
+
+      @client.indices.create index: 'test2'
+
+      @client.cluster.health wait_for_status: 'yellow'
+    end
+
+    teardown do
+      @client.indices.delete index: '_all'
+    end
+
+    should "copy documents from one index to another" do
+      reindex = Elasticsearch::Extensions::Reindex.new \
+                  source: { index: 'test1', client: @client },
+                  dest: { index: 'test2' },
+                  batch_size: 2,
+                  refresh: true
+
+      result = reindex.perform
+
+      assert_equal 0, result[:errors]
+      assert_equal 3, @client.search(index: 'test2')['hits']['total']['value']
+    end
+
+    should "transform documents with a lambda" do
+      reindex = Elasticsearch::Extensions::Reindex.new \
+                  source: { index: 'test1', client: @client },
+                  dest: { index: 'test2' },
+                  transform: lambda { |d| d['_source']['category'].upcase! },
+                  refresh: true
+
+      result = reindex.perform
+
+      assert_equal 0, result[:errors]
+      assert_equal 3, @client.search(index: 'test2')['hits']['total']['value']
+      assert_equal 'ONE', @client.get(index: 'test2', type: 'd', id: 1)['_source']['category']
+    end
+
+    should "return the number of errors" do
+      @client.indices.create index: 'test3', body: { mappings: { properties: { category: { type: 'integer' } }}}
+      @client.cluster.health wait_for_status: 'yellow'
+
+      reindex = Elasticsearch::Extensions::Reindex.new \
+                  source: { index: 'test1', client: @client },
+                  dest: { index: 'test3' }
+
+      result = reindex.perform
+
+      @client.indices.refresh index: 'test3'
+
+      assert_equal 3, result[:errors]
+      assert_equal 0, @client.search(index: 'test3')['hits']['total']['value']
+    end
+
+    should "reindex via the API integration" do
+      @client.indices.create index: 'test4'
+
+      @client.reindex source: { index: 'test1' }, dest: { index: 'test4' }
+      @client.indices.refresh index: 'test4'
+
+      assert_equal 3, @client.search(index: 'test4')['hits']['total']['value']
+    end
+  end
+
+end

data/test/reindex/unit/reindex_test.rb

@@ -0,0 +1,123 @@
+# Licensed to Elasticsearch B.V. under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch B.V. licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require 'test_helper'
+require 'elasticsearch/extensions/reindex'
+
+class Elasticsearch::Extensions::ReindexTest < Elasticsearch::Test::UnitTestCase
+  context "The Reindex extension module" do
+    DEFAULT_OPTIONS = { source: { index: 'foo', client: Object.new }, dest: { index: 'bar' } }
+
+    should "require options" do
+      assert_raise ArgumentError do
+        Elasticsearch::Extensions::Reindex.new
+      end
+    end
+
+    should "allow to initialize the class" do
+      assert_instance_of Elasticsearch::Extensions::Reindex::Reindex,
+                         Elasticsearch::Extensions::Reindex.new(DEFAULT_OPTIONS)
+    end
+
+    should "add the reindex to the API and client" do
+      assert_includes Elasticsearch::API::Actions.public_instance_methods.sort, :reindex
+      assert_respond_to Elasticsearch::Client.new, :reindex
+    end
+
+    should "pass the client when used in API mode" do
+      client = Elasticsearch::Client.new
+
+      Elasticsearch::Extensions::Reindex::Reindex
+        .expects(:new)
+        .with({source: { client: client }})
+        .returns(stub perform: {})
+
+      client.reindex
+    end
+
+    context "when performing the operation" do
+      setup do
+        d = { '_id' => 'foo', '_type' => 'type', '_source' => { 'foo' => 'bar' } }
+        @default_response = { 'hits' => { 'hits' => [d] } }
+        @empty_response   = { 'hits' => { 'hits' => [] } }
+        @bulk_request     = [{ index: {
+                                '_index' => 'bar',
+                                '_type'  => d['_type'],
+                                '_id'    => d['_id'],
+                                'data'   => d['_source']
+                               } }]
+        @bulk_response    = {'errors'=>false, 'items' => [{'index' => {}}, {'index' => {}}]}
+        @bulk_response_error = {'errors'=>true, 'items' => [{'index' => {}}, {'index' => {'error' => 'FOOBAR'}}]}
+      end
+
+      should "scroll through the index and save batches in bulk" do
+        client  = mock()
+        subject = Elasticsearch::Extensions::Reindex.new source: { index: 'foo', client: client },
+                                                         dest: { index: 'bar' }
+
+        client.expects(:search)
+          .returns({ '_scroll_id' => 'scroll_id_1' }.merge(Marshal.load(Marshal.dump(@default_response))))
+        client.expects(:scroll)
+          .returns(Marshal.load(Marshal.dump(@default_response)))
+          .then
+          .returns(@empty_response).times(2)
+        client.expects(:bulk)
+          .with(body: @bulk_request)
+          .returns(@bulk_response).times(2)
+
+        result = subject.perform
+
+        assert_equal 0, result[:errors]
+      end
+
+      should "return the number of errors" do
+        client  = mock()
+        subject = Elasticsearch::Extensions::Reindex.new source: { index: 'foo', client: client },
+                                                         dest: { index: 'bar' }
+
+        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' }.merge(@default_response))
+        client.expects(:scroll).returns(@empty_response)
+        client.expects(:bulk).with(body: @bulk_request).returns(@bulk_response_error)
+
+        result = subject.perform
+
+        assert_equal 1, result[:errors]
+      end
+
+      should "transform the documents with a lambda" do
+        client  = mock()
+        subject = Elasticsearch::Extensions::Reindex.new \
+          source: { index: 'foo', client: client },
+          dest: { index: 'bar' },
+          transform: lambda { |d| d['_source']['foo'].upcase!; d }
+
+        client.expects(:search).returns({ '_scroll_id' => 'scroll_id_1' }.merge(@default_response))
+        client.expects(:scroll).returns(@empty_response)
+        client.expects(:bulk).with do |arguments|
+                assert_equal 'BAR', arguments[:body][0][:index]['data']['foo']
+                true
+              end
+              .returns(@bulk_response)
+
+        result = subject.perform
+
+        assert_equal 0, result[:errors]
+      end
+    end
+
+  end
+end