elasticsearch-extensions 0.0.15 → 0.0.16

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NzFhMWNhNDFlNjBmMjcxMzAyMTVlZDRkZWFlYjI2NzkzNzc0NjE2Nw==
-  data.tar.gz: !binary |-
-    NDU0NzFlZjRjYjU1M2RlZDdkNGUxMzMyYmUxMzc2MzhlYWExODdmNw==
+SHA1:
+  metadata.gz: b3a84e8764023e5bf5d1ea21d93308eb3bfeafa0
+  data.tar.gz: 49f72a1364a104f0fbbe8fcf6a4986e4dd88121f
 SHA512:
-  metadata.gz: !binary |-
-    NDNjNDdmN2E4ZjZhZDdjYzkxZDJjYTZlMThkOWYwZDQyODM2ZjhmZTllMTI2
-    NTgwMjllZjNhOTYzNGZiODg4YWNkODMzZjIyNDVkMGQyNTJkZTFiNTY1NTVi
-    NGQ2NDY4NTI2YjQyZjQxYzU1Y2JkM2E5ODZiNTUyMzRjZjQ4YTE=
-  data.tar.gz: !binary |-
-    N2JjZWE5NmJjYTk3YzU4YWQ3ODUwN2NjZDQ2Njg4MmM0NzQ4ODMwNTkxZDQ2
-    NmNiNTc0ODQ1OTNlMGQwZTIyNDQwNGI0ZWJmZjFjYmYzYWUzOWVmNzFjNWQ1
-    NzJmMDljNjk2NjExNGIwZmMzNDk2MGNjNTE1YjkzNjNjYWNhNTg=
+  metadata.gz: 7ac3d9463af301fbb315573d56217d6c1973d9f3e53a15544acaccaf51fe50ac3167583c255182e2176ea98fe2e56f3da4e1981f443ea653c144ebbb10c074b5
+  data.tar.gz: a9834e293e11e73a624978e16ec2a14b7ca540e0e0a04d33e9ac511f991508105cf6cd8e2c39dc6aa93d10a4322a50f749f750d500a6810d8ad05cdc653d886f

data/README.md

@@ -161,6 +161,22 @@ or install it from a source code checkout:
     bundle install
     rake install
 
+## Development
+
+To work on the code, clone and bootstrap the main repository first --
+please see instructions in the main [README](../README.md#development).
+
+To run tests, launch a testing cluster -- again, see instructions
+in the main [README](../README.md#development) -- and use the Rake tasks:
+
+```
+time rake test:unit
+time rake test:integration
+```
+
+Unit tests have to use Ruby 1.8 compatible syntax, integration tests
+can use Ruby 2.x syntax and features.
+
 ## License
 
 This software is licensed under the Apache 2 license, quoted below.

data/elasticsearch-extensions.gemspec

@@ -38,7 +38,8 @@ Gem::Specification.new do |s|
   s.add_development_dependency "mocha"
   s.add_development_dependency "turn"
   s.add_development_dependency "yard"
-  s.add_development_dependency "ci_reporter"
+  s.add_development_dependency "pry"
+  s.add_development_dependency "ci_reporter", "~> 1.9"
 
   if defined?(RUBY_VERSION) && RUBY_VERSION < '1.9'
     s.add_development_dependency "json"
@@ -49,4 +50,9 @@ Gem::Specification.new do |s|
     s.add_development_dependency "simplecov-rcov"
     s.add_development_dependency "cane"
   end
+
+  # Gems for testing integrations
+  s.add_development_dependency "backup", '~> 4'
+  s.add_development_dependency "patron"
+  s.add_development_dependency "oj"
 end

data/lib/elasticsearch/extensions/backup.rb

@@ -0,0 +1,142 @@
+# encoding: utf-8
+
+require 'pathname'
+require 'fileutils'
+
+require 'multi_json'
+require 'oj'
+
+require 'elasticsearch'
+require 'patron'
+
+module Backup
+  module Database
+
+    # Integration with the Backup gem [https://github.com/meskyanichi/backup/]
+    #
+    # This extension allows to backup Elasticsearch indices as flat JSON files on the disk.
+    #
+    # Use the Backup gem's DSL to configure the backup:
+    #
+    #     require 'elasticsearch/extensions/backup'
+    #
+    #     Model.new(:elasticsearch_backup, 'Elasticsearch') do
+    #
+    #       database Elasticsearch do |db|
+    #         # db.url     = 'http://localhost:9200'
+    #         # db.indices = 'articles,people'
+    #         # db.size    = 500
+    #         # db.scroll  = '10m'
+    #       end
+    #
+    #       store_with Local do |local|
+    #         local.path = '/usr/local/var/backups'
+    #         local.keep = 3
+    #       end
+    #
+    #       compress_with Gzip
+    #     end
+    #
+    # Perform the backup with the Backup gem's command line utility:
+    #
+    #     $ backup perform -t elasticsearch_backup
+    #
+    #
+    # A simple recover script could look like this:
+    #
+    #     PATH = '/path/to/backup/'
+    #
+    #     require 'elasticsearch'
+    #     client  = Elasticsearch::Client.new log: true
+    #     payload = []
+    #
+    #     Dir[ File.join( PATH, '**', '*.json' ) ].each do |file|
+    #       document = MultiJson.load(File.read(file))
+    #       item = document.merge(data: document['_source'])
+    #       document.delete('_source')
+    #       document.delete('_score')
+    #
+    #       payload << { index: item }
+    #
+    #       if payload.size == 100
+    #         client.bulk body: payload
+    #         payload = []
+    #       end
+    #
+    #       client.bulk body: payload
+    #     end
+    #
+    # @see http://meskyanichi.github.io/backup/v4/
+    #
+    class Elasticsearch < Base
+      class Error < ::Backup::Error; end
+
+      attr_accessor :url,
+                    :indices,
+                    :size,
+                    :scroll
+
+      attr_accessor :mode
+
+      def initialize(model, database_id = nil, &block)
+        super
+
+        @url     ||= 'http://localhost:9200'
+        @indices ||= '_all'
+        @size    ||= 100
+        @scroll  ||= '10m'
+        @mode    ||= 'single'
+
+        instance_eval(&block) if block_given?
+      end
+
+      def perform!
+        super
+
+        case mode
+          when 'single'
+            __perform_single
+          else
+            raise Error, "Unsupported mode [#{mode}]"
+        end
+
+        log!(:finished)
+      end
+
+      def client
+        @client ||= ::Elasticsearch::Client.new url: url, logger: logger
+      end
+
+      def path
+        Pathname.new File.join(dump_path , dump_filename.downcase)
+      end
+
+      def logger
+        logger = Backup::Logger.__send__(:logger)
+        logger.instance_eval do
+          def debug(*args);end
+          # alias :debug :info
+          alias :fatal :warn
+        end
+        logger
+      end
+
+
+      def __perform_single
+        r = client.search index: indices, search_type: 'scan', scroll: scroll, size: size
+        raise Error, "No scroll_id returned in response:\n#{r.inspect}" unless r['_scroll_id']
+
+        while r = client.scroll(scroll_id: r['_scroll_id'], scroll: scroll) and not r['hits']['hits'].empty? do
+          r['hits']['hits'].each do |hit|
+            FileUtils.mkdir_p "#{path.join hit['_index'], hit['_type']}"
+            File.open("#{path.join hit['_index'], hit['_type'], hit['_id']}.json", 'w') do |file|
+              file.write MultiJson.dump(hit)
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+::Backup::Config::DSL::Elasticsearch = ::Backup::Database::Elasticsearch

data/lib/elasticsearch/extensions/test/cluster.rb

@@ -76,7 +76,7 @@ module Elasticsearch
           arguments[:es_params]    ||= ENV['TEST_CLUSTER_PARAMS'] || ''
           arguments[:path_work]    ||= '/tmp'
           arguments[:node_name]    ||= 'node'
-          arguments[:timeout]      ||= 30
+          arguments[:timeout]      ||= (ENV['TEST_CLUSTER_TIMEOUT'] || 30).to_i
 
           if running? :on => arguments[:port], :as => arguments[:cluster_name]
             print "[!] Elasticsearch cluster already running".ansi(:red)
@@ -102,11 +102,13 @@ module Elasticsearch
                 -D es.index.store.type=#{arguments[:index_store]} \
                 -D es.path.data=#{arguments[:path_data]} \
                 -D es.path.work=#{arguments[:path_work]} \
+                -D es.cluster.routing.allocation.disk.threshold_enabled=false \
                 -D es.network.host=0.0.0.0 \
                 -D es.discovery.zen.ping.multicast.enabled=true \
                 -D es.script.disable_dynamic=false \
                 -D es.node.test=true \
                 -D es.node.bench=true \
+                -D es.logger.level=DEBUG \
                 #{arguments[:es_params]} \
                 > /dev/null
             COMMAND

data/lib/elasticsearch/extensions/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Extensions
-    VERSION = "0.0.15"
+    VERSION = "0.0.16"
   end
 end

data/test/backup/unit/backup_test.rb

@@ -0,0 +1,75 @@
+require 'test_helper'
+
+require 'backup'
+
+require 'elasticsearch/extensions/backup'
+
+class Elasticsearch::Extensions::BackupTest < Test::Unit::TestCase
+  context "The Backup gem extension" do
+    setup do
+      @model = ::Backup::Model.new(:test_trigger, 'test label')
+      @subject = ::Backup::Database::Elasticsearch.new(@model)
+    end
+
+    should "have a client" do
+      assert_instance_of Elasticsearch::Transport::Client, @subject.client
+    end
+
+    should "have a path" do
+      assert_instance_of Pathname, @subject.path
+    end
+
+    should "have defaults" do
+      assert_equal 'http://localhost:9200', @subject.url
+      assert_equal '_all', @subject.indices
+    end
+
+    should "be configurable" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) do |db|
+        db.url = 'https://example.com'
+        db.indices = 'foo,bar'
+      end
+
+      assert_equal 'https://example.com', @subject.url
+      assert_equal 'foo,bar', @subject.indices
+
+      assert_equal 'example.com', @subject.client.transport.connections.first.host[:host]
+    end
+
+    should "perform the backup" do
+      @subject.expects(:__perform_single)
+      @subject.perform!
+    end
+
+    should "raise an expection for an unsupported type of backup" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) { |db| db.mode = 'foobar' }
+      assert_raise ::Backup::Database::Elasticsearch::Error do
+        @subject.perform!
+      end
+    end
+
+    should "scan and scroll the index" do
+      @subject = ::Backup::Database::Elasticsearch.new(@model) { |db| db.indices = 'test' }
+
+      @subject.client
+        .expects(:search)
+        .with do |params|
+          assert_equal 'test', params[:index]
+        end
+        .returns({"_scroll_id" => "abc123"})
+
+      @subject.client
+        .expects(:scroll)
+        .twice
+        .returns({"_scroll_id" => "def456",
+                  "hits" => { "hits" => [ {"_index"=>"test", "_type"=>"doc", "_id"=>"1", "_source"=>{"title"=>"Test"}} ] }
+                })
+        .then
+        .returns({"_scroll_id" => "ghi789",
+                  "hits" => { "hits" => [] }
+                })
+
+      @subject.__perform_single
+    end
+  end
+end

metadata

@@ -1,209 +1,265 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-extensions
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.16
 platform: ruby
 authors:
 - Karel Minarik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-22 00:00:00.000000000 Z
+date: 2014-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
 - !ruby/object:Gem::Dependency
   name: ruby-prof
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: shoulda-context
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: turn
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ci_reporter
+  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ci_reporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov-rcov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: cane
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: backup
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4'
+- !ruby/object:Gem::Dependency
+  name: patron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Extensions for the Elasticsearch Rubygem
@@ -213,7 +269,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -225,12 +281,14 @@ files:
 - lib/elasticsearch/extensions/ansi/actions.rb
 - lib/elasticsearch/extensions/ansi/helpers.rb
 - lib/elasticsearch/extensions/ansi/response.rb
+- lib/elasticsearch/extensions/backup.rb
 - lib/elasticsearch/extensions/test/cluster.rb
 - lib/elasticsearch/extensions/test/cluster/tasks.rb
 - lib/elasticsearch/extensions/test/profiling.rb
 - lib/elasticsearch/extensions/test/startup_shutdown.rb
 - lib/elasticsearch/extensions/version.rb
 - test/ansi/unit/ansi_test.rb
+- test/backup/unit/backup_test.rb
 - test/test_helper.rb
 homepage: ''
 licenses:
@@ -242,12 +300,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -258,5 +316,6 @@ specification_version: 4
 summary: Extensions for the Elasticsearch Rubygem
 test_files:
 - test/ansi/unit/ansi_test.rb
+- test/backup/unit/backup_test.rb
 - test/test_helper.rb
 has_rdoc: