eson-more 0.8.0

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright (c) 2011 Florian Gilcher <florian.gilcher@asquera.de>, Felix Gilcher <felix.gilcher@asquera.de>
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,16 @@
+# eson-more. Handy functions for ElasticSearch
+
+eson-more implements things you don't necessarily need, but which come in handy from time to time.
+
+eson-more is in an experimental stage, so please don't rely on details.
+
+## Usage
+
+    require 'eson-more/all'
+
+    c = Eson::HTTP::Client.new
+
+    # print the whole index in chunks
+    c.all(:index => "default") do |results|
+      puts results
+    end

data/Rakefile

@@ -0,0 +1,31 @@
+require 'rake/testtask'
+require 'rubygems/package_task'
+require 'rake/testtask'
+
+def gemspec
+  @gemspec ||= begin
+    file = File.expand_path("eson-more.gemspec")
+    ::Gem::Specification.load(file)
+  end
+end
+
+desc "Validates the gemspec"
+task :gemspec do
+  gemspec.validate
+end
+
+Gem::PackageTask.new(gemspec) do |pkg|
+  pkg.gem_spec = gemspec
+end
+
+task :package => :gemspec
+
+Rake::TestTask.new(:test) do |test|
+  test.pattern = 'test/**/*_test.rb'
+  
+  test.verbose = true
+end
+
+task :seed do
+  require './test/seeds/seeds'
+end

data/eson-more.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+#require "./lib/elsearch"
+
+Gem::Specification.new do |s|
+  s.name        = "eson-more"
+  s.version     = "0.8.0"
+
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Florian Gilcher"]
+  s.email       = ["florian.gilcher@asquera.de"]
+  s.homepage    = ""
+  s.summary     = %q{A modular client for ElasticSearch - additional functions}
+  s.description = %q{A modular client for ElasticSearch. It provides
+  an implementation of the Query language as well as multiple client implementations
+  for HTTP and native access.}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  #s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency "eson-core"
+  s.add_development_dependency "elasticsearch-node"
+  s.add_development_dependency "riot"
+end

data/lib/eson-more.rb

@@ -0,0 +1,6 @@
+require 'eson-core'
+
+require_relative 'eson/more/extract'
+require_relative 'eson/more/all'
+require_relative 'eson/more/transplant'
+require_relative 'eson/more/reindex'

data/lib/eson/more/all.rb

@@ -0,0 +1,46 @@
+module Eson
+  module More
+    module All
+      DEFAULT_OPTS = {:search_type => :scan, :scroll => "10m", :size => 50}
+
+      module Functions
+        def self.fetch_more_results(client, scroll_id)
+          result = client.scroll(:scroll_id => scroll_id, :scroll => "10m")
+          scroll_id = result["_scroll_id"]
+
+          results = client.extract_hits(result)
+          scroll_id = nil if results.empty?
+
+          return scroll_id, results
+        end
+      end
+
+      # Retrieves all documents matching a given option set. Automatically 
+      # handles scrolling.
+      # 
+      # @param [Hash] opts the options to retrieve documents
+      def all(opts)
+        scroll_response = search(DEFAULT_OPTS.merge(opts))
+        scroll_id = scroll_response["_scroll_id"]
+
+        if block_given?
+          while scroll_id
+            scroll_id, results = Functions.fetch_more_results(self, scroll_id)
+            yield results
+          end
+        else
+          docs = []
+          while scroll_id
+            scroll_id, results = Functions.fetch_more_results(self, scroll_id)
+            docs += results
+          end
+          docs
+        end
+      end
+    end
+  end
+end
+
+Eson::Client.class_eval {
+  include Eson::More::All
+}

data/lib/eson/more/extract.rb

@@ -0,0 +1,23 @@
+module Eson
+  module More
+    module Extract
+      # Extracts doc sources as hashes from a search result.
+      # 
+      # @param [Hash] response the response body as returned by a search request.
+      def extract_sources(response)
+        extract_hits(response).map {|h| h["_source"] }
+      end
+
+      # Extracts doc hits as hashes from a search result.
+      # 
+      # @param [Hash] response the response body as returned by a search request.
+      def extract_hits(response)
+        response["hits"]["hits"]
+      end
+    end
+  end
+end
+
+Eson::Client.class_eval {
+  include Eson::More::Extract
+}

data/lib/eson/more/reindex.rb

@@ -0,0 +1,29 @@
+module Eson
+  module More
+    module Reindex
+      # Reindex a whole index by scrolling the old one and writing to the 
+      # new one using a bulk request.
+      # 
+      # @param [String] from the index to reindex from
+      # @param [String] to the index to reindex to
+      def reindex(from, to)
+        all(:index => from) do |chunk|
+          if chunk.size > 0
+            bulk do |b|
+              chunk.each do |doc|
+                b.index :index => to,
+                        :type => doc["_type"],
+                        :id => doc["_id"],
+                        :doc => doc["_source"]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+Eson::Client.class_eval {
+  include Eson::More::Reindex
+}

data/lib/eson/more/transplant.rb

@@ -0,0 +1,101 @@
+module Eson
+  module More
+    module Transplant
+      module Functions
+        def self.extract_ids(*args)
+          args.map do |d|
+            case d
+            when String, Fixnum
+              d
+            when Hash
+              d[:id] || d["id"]
+            when nil
+              d
+            else
+              d.id
+            end
+          end
+        end
+
+        def self.get_doc(client, old_parent, doc)
+          case doc
+          when Hash
+            doc
+          when String
+            client.get(:id => doc)
+          else
+            doc.to_h
+          end
+        end
+
+        def self.get_children(client, parent)
+          client.all(:type => nil, :q => "_parent:#{parent}",:routing => parent)
+        end
+
+        def self.get_doc(client, parent, doc)
+          client.extract_hits(
+            client.search(:type => nil, :q => "_id:#{doc}", :routing => parent)
+          )
+        end
+      end
+
+      # Transplants a document from one parent to another by deleting the old
+      # one and reindexing it with a new parent. If no doc is given, all
+      # documents of the old parent will be transplanted.
+      # 
+      # To change the index and parent type this command operates on, 
+      # use Client#with:
+      #   
+      #   client.with :index => "foo", :type => "blog" do |c|
+      #     c.transplant(1, 2, 3)
+      #   end
+      # 
+      # Transplant does not refresh automatically.
+      # 
+      # @param [String, Hash, Object#id] old_parent the old parent document,
+      #         either as an Object from which the ID can be retrieved.
+      # @param [String, Hash, Object#id] new_parent the new parent document,
+      #         either as an Object from which the ID can be retrieved.
+      # @param [String, Hash, #id, #to_h] doc the document to transplant,
+      #         either as an Object from which the ID can be retrieved.
+      def transplant(old_parent, new_parent, doc = nil)
+        old_pid, new_pid, doc_id = Functions.extract_ids(old_parent, new_parent, doc)
+
+        if doc.nil?
+          hits = Functions.get_children(self, old_pid)
+        else
+          hits = Functions.get_doc(self, old_pid, doc)
+        end
+
+        unless hits.empty?
+          bulk do |b|
+            hits.each do |hit|
+              b.delete :type => hit["_type"],
+                       :id => hit["_id"],
+                       :routing => old_pid
+              b.index  :type => hit["_type"],
+                       :id => hit["_id"],
+                       :parent => new_pid,
+                       :doc => hit["_source"]
+            end
+          end
+        end
+      end
+
+      # Like #transplant, but removes the old parent.
+      # 
+      # @param [String, Hash, Object#id] old_parent the old parent document,
+      #         either as an Object from which the ID can be retrieved.
+      # @param [String, Hash, Object#id] new_parent the new parent document,
+      #         either as an Object from which the ID can be retrieved.
+      def merge_parents(old_parent, new_parent)
+        transplant(old_parent, new_parent)
+        delete(:id => old_parent)
+      end
+    end
+  end
+end
+
+Eson::Client.class_eval {
+  include Eson::More::Transplant
+}

data/log4j.properties

@@ -0,0 +1,18 @@
+log4j.rootLogger=INFO, out
+log4j.logger.jgroups=WARN
+
+#log4j.logger.discovery=TRACE
+#log4j.logger.cluster=TRACE
+#log4j.logger.indices.cluster=DEBUG
+#log4j.logger.index=TRACE
+#log4j.logger.index.engine=DEBUG
+#log4j.logger.index.shard=TRACE
+#log4j.logger.index.cache=DEBUG
+#log4j.logger.http=TRACE
+#log4j.logger.monitor.jvm=DEBUG
+#log4j.logger.cluster.action.shard=TRACE
+#log4j.logger.index.gateway=TRACE
+
+log4j.appender.out=org.apache.log4j.ConsoleAppender
+log4j.appender.out.layout=org.apache.log4j.PatternLayout
+log4j.appender.out.layout.ConversionPattern=[%d{ABSOLUTE}][%-5p][%-25c] %m%n

data/test/more/all_test.rb

@@ -0,0 +1,17 @@
+require './test/test_config'
+
+context "All" do
+ helper(:node) { Node::External.instance }
+  
+  helper(:client) do
+    Eson::HTTP::Client.new(:server => "http://#{node.ip}:#{node.port}",
+                           :logger => 'test/test.log')
+  end
+  
+  setup do
+    client.all(:index => "all")
+  end
+
+  asserts(:class).equals(Array)
+  asserts(:length).equals(500)
+end

data/test/more/extract_source_test.rb

@@ -0,0 +1,28 @@
+require './test/test_config'
+
+context "Extract" do
+  helper(:node) { Node::External.instance }
+  
+  helper(:client) do
+    Eson::HTTP::Client.new(:server => "http://#{node.ip}:#{node.port}",
+                           :logger => 'test/test.log')
+  end
+  
+  describe "extract_sources" do
+    setup do
+      client.extract_sources(client.simple_search :index => "extract_source", :q => "*")
+    end
+    
+    asserts(:class).equals(Array)
+    asserts(:first).equals("foo" => "bar")
+  end
+  
+  describe "extract_hits" do
+    setup do
+      client.extract_hits(client.simple_search :index => "extract_source", :q => "*")
+    end
+    
+    asserts(:class).equals(Array)
+    asserts("index of first element") { topic.first["_index"]}.equals("extract_source")
+  end
+end

data/test/more/reindex_test.rb

@@ -0,0 +1,19 @@
+require './test/test_config'
+
+context "Reindex" do
+ helper(:node) { Node::External.instance }
+  
+  helper(:client) do
+    Eson::HTTP::Client.new(:server => "http://#{node.ip}:#{node.port}",
+                           :logger => 'test/test.log')
+  end
+  
+  setup do
+    client.reindex("all", "to_all")
+    client.refresh(:index => "to_all")
+    client.all(:index => "to_all")
+  end
+
+  asserts(:class).equals(Array)
+  asserts(:length).equals(500)
+end

data/test/more/transplant_test.rb

@@ -0,0 +1,46 @@
+require './test/test_config'
+
+context "Transplant" do
+  helper(:node) { Node::External.instance }
+
+  helper(:client) do
+    Eson::HTTP::Client.new(:server => "http://#{node.ip}:#{node.port}",
+                           :logger => 'test/test.log')
+  end
+
+  describe "transplant one document" do
+    setup do
+      client.with :index => "transplant" do |c|
+        c.transplant(1, 2, 4)
+        c.refresh
+        c.get(:type => "blog_tag", :id => 4, :routing => 2)
+      end
+    end
+    asserts(:source) { topic["_source"]}.equals({"foo" => "bar"})
+  end
+
+  describe "transplant all" do
+    setup do
+      client.with :index => "transplant" do |c|
+        c.transplant(1,2)
+        c.refresh
+        c.all(:q => "_parent:2")
+      end
+    end
+    asserts(:class).equals(Array)
+    asserts(:length).equals(498)
+  end
+
+  describe "merge_parents" do
+    setup do
+      client.with :index => "transplant", :type => "blog" do |c|
+        c.merge_parents(1,2)
+        c.refresh
+        c
+      end
+    end
+
+    asserts("retrieving the old parent") { topic.get(:id => 1) }.raises(Eson::NotFoundError)
+    asserts("count of all children of the new client") { topic.all(:type => "blog_tag", :q => "_parent:2").length }.equals(498)
+  end
+end

data/test/seeds/seeds.rb

@@ -0,0 +1,70 @@
+# basically, this is also a test for the bulk interface :)
+
+node = Node::External.instance
+
+c = Eson::Client.new(:server => "http://#{node.ip}:#{node.port}", 
+                     :protocol => Eson::HTTP, 
+                     :plugins => [Eson::ResponseParser], 
+                     :logger => 'test/test.log')
+
+c.delete_index :index => "_all" rescue nil
+
+c.bulk do |bulk_request|
+  1.upto(500) do |i|
+    bulk_request.index :index => "extract_source",
+                       :type => "bar",
+                       :doc => {"foo" => "bar"}
+  end
+end
+
+c.bulk do |bulk_request|
+  1.upto(500) do |i|
+    bulk_request.index :index => "all",
+                       :type => "bar",
+                       :doc => {"foo" => "bar"}
+  end
+end
+
+c.create_index :index => "transplant"
+c.put_mapping :index => "transplant",
+              :type => 'blog',
+              :mapping => {
+                :blog => {
+                  :properties => {
+                    :foo => { :type => "string", :index => "not_analyzed" }
+                  }
+                }
+              }
+
+c.put_mapping :index => "transplant",
+              :type => 'blog_tag',
+              :mapping => {
+                :blog_tag => {
+                  :_parent => { :type => "blog" }
+                }
+              }
+
+c.index :index => "transplant",
+        :type => :blog,
+        :id => 1,
+        :doc => {:foo => "bar"}
+c.index :index => "transplant",
+        :type => :blog,
+        :id => 2,
+        :doc => {:foo => "bar"}
+
+c.refresh :index => "transplant"
+
+c.bulk do |bulk_request|
+  3.upto(500) do |i|
+    bulk_request.index :index  => "transplant",
+                       :type   => "blog_tag",
+                       :id     => i,
+                       :doc    => {"foo" => "bar"},
+                       :parent => ((i % 2) + 1)
+  end
+end
+
+c.refresh(:index => "all")
+c.refresh(:index => "transplant")
+c.refresh(:index => "extract_source")

data/test/test_config.rb

@@ -0,0 +1,34 @@
+begin
+  # Require the preresolved locked set of gems.
+  require File.expand_path('../../.bundle/environment', __FILE__)
+rescue LoadError
+  # Fallback on doing the resolve at runtime.
+  require 'rubygems'
+  require 'bundler'
+  Bundler.setup
+end
+
+Bundler.require(:test, :default)
+
+require 'eson-http'
+require 'eson-more'
+
+require 'elasticsearch-node/external'
+
+module Node
+  module External
+    def self.instance
+      @node ||= begin
+        node = ElasticSearch::Node::External.new("gateway.type" => "none")
+        at_exit do
+          node.close
+        end
+        node
+      end
+    end
+  end
+end
+
+Node::External.instance
+require 'riot'
+require './test/seeds/seeds'

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: eson-more
+version: !ruby/object:Gem::Version
+  version: 0.8.0
+  prerelease: 
+platform: ruby
+authors:
+- Florian Gilcher
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eson-core
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: elasticsearch-node
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: riot
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "A modular client for ElasticSearch. It provides\n  an implementation
+  of the Query language as well as multiple client implementations\n  for HTTP and
+  native access."
+email:
+- florian.gilcher@asquera.de
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.md
+- README.md
+- Rakefile
+- eson-more.gemspec
+- lib/eson-more.rb
+- lib/eson/more/all.rb
+- lib/eson/more/extract.rb
+- lib/eson/more/reindex.rb
+- lib/eson/more/transplant.rb
+- log4j.properties
+- test/more/all_test.rb
+- test/more/extract_source_test.rb
+- test/more/reindex_test.rb
+- test/more/transplant_test.rb
+- test/seeds/seeds.rb
+- test/test_config.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.21
+signing_key: 
+specification_version: 3
+summary: A modular client for ElasticSearch - additional functions
+test_files:
+- test/more/all_test.rb
+- test/more/extract_source_test.rb
+- test/more/reindex_test.rb
+- test/more/transplant_test.rb
+- test/seeds/seeds.rb
+- test/test_config.rb