elastomer-client 0.4.1 → 0.5.0

data/lib/elastomer/client/repository.rb

@@ -35,7 +35,7 @@ module Elastomer
           raise exception
         end
       end
-      alias :exist? :exists?
+      alias_method :exist?, :exists?
 
       # Create the repository.
       # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-snapshots.html#_repositories
@@ -103,7 +103,7 @@ module Elastomer
       def snapshot(snapshot = nil)
         client.snapshot(name, snapshot)
       end
-      alias :snapshots :snapshot
+      alias_method :snapshots, :snapshot
 
       # Internal: Add default parameters to the `params` Hash and then apply
       # `overrides` to the params if any are given.

data/lib/elastomer/client/scroller.rb

@@ -0,0 +1,235 @@
+module Elastomer
+  class Client
+
+    # Create a new Scroller instance for scrolling all results from a `query`.
+    #
+    # query  - The query to scroll as a Hash or a JSON encoded String
+    # opts   - Options Hash
+    #   :index  - the name of the index to search
+    #   :type   - the document type to search
+    #   :scroll - the keep alive time of the scrolling request (5 minutes by default)
+    #   :size   - the number of documents per shard to fetch per scroll
+    #
+    # Examples
+    #
+    #   scroll = client.scroll('{"query":{"match_all":{}}}', :index => 'test')
+    #   scroll.each_document do |document|
+    #     document['_id']
+    #     document['_source']
+    #   end
+    #
+    # Returns a new Scroller instance
+    def scroll( query, opts = {} )
+      Scroller.new(self, query, opts)
+    end
+
+    # Create a new Scroller instance for scrolling all results from a `query`
+    # via "scan" semantics.
+    #
+    # query  - The query to scan as a Hash or a JSON encoded String
+    # opts   - Options Hash
+    #   :index  - the name of the index to search
+    #   :type   - the document type to search
+    #   :scroll - the keep alive time of the scrolling request (5 minutes by default)
+    #   :size   - the number of documents per shard to fetch per scroll
+    #
+    # Examples
+    #
+    #   scan = client.scan('{"query":{"match_all":{}}}', :index => 'test')
+    #   scan.each_document do |document|
+    #     document['_id']
+    #     document['_source']
+    #   end
+    #
+    # Returns a new Scroller instance
+    def scan( query, opts = {} )
+      opts = opts.merge(:search_type => 'scan')
+      Scroller.new(self, query, opts)
+    end
+
+    # Begin scrolling a query.
+    # See http://www.elasticsearch.org/guide/reference/api/search/scroll/
+    #
+    # opts   - Options Hash
+    #   :query       - the query to scroll as a Hash or JSON encoded String
+    #   :index       - the name of the index to search
+    #   :type        - the document type to search
+    #   :scroll      - the keep alive time of the scrolling request (5 minutes by default)
+    #   :size        - the number of documents per shard to fetch per scroll
+    #   :search_type - set to 'scan' for scan semantics
+    #
+    # Examples
+    #
+    #   h = client.start_scroll('{"query":{"match_all":{}},"sort":{"created":"desc"}}', :index => 'test')
+    #   scroll_id = h['_scroll_id']
+    #   h['hits']['hits'].each { |doc| ... }
+    #
+    #   h = client.continue_scroll(scroll_id)
+    #   scroll_id = h['_scroll_id']
+    #   h['hits']['hits'].each { |doc| ... }
+    #
+    #   # repeat until there are no more hits
+    #
+    # Returns the response body as a Hash.
+    def start_scroll( opts = {} )
+      opts = opts.merge :action => 'search.start_scroll'
+      response = get '{/index}{/type}/_search', opts
+      response.body
+    end
+
+    # Continue scrolling a query.
+    # See http://www.elasticsearch.org/guide/reference/api/search/scroll/
+    #
+    # scroll_id - The current scroll ID as a String
+    # scroll    - The keep alive time of the scrolling request (5 minutes by default)
+    #
+    # Examples
+    #
+    #   scroll_id = client.start_scroll('{"query":{"match_all":{}}}', :index => 'test', :search_type => 'scan')['_scroll_id']
+    #
+    #   h = client.continue_scroll scroll_id   # scroll to get the next set of results
+    #   scroll_id = h['_scroll_id']            # and store the scroll_id to use later
+    #
+    #   h = client.continue_scroll scroll_id   # scroll again to get the next set of results
+    #   scroll_id = h['_scroll_id']            # and store the scroll_id to use later
+    #
+    #   # repeat until the results are empty
+    #
+    # Returns the response body as a Hash.
+    def continue_scroll( scroll_id, scroll = '5m' )
+      response = get '/_search/scroll', :body => scroll_id, :scroll => scroll, :action => 'search.scroll'
+      response.body
+    end
+
+    class Scroller
+      # Create a new scroller that can be used to iterate over all the documents
+      # returned by the `query`. The Scroller supports both the 'scan' and the
+      # 'scroll' search types.
+      #
+      # See http://www.elasticsearch.org/guide/reference/api/search/scroll/
+      # and the "Scan" section of http://www.elasticsearch.org/guide/reference/api/search/search-type/
+      #
+      # client - Elastomer::Client used for HTTP requests to the server
+      # query  - The query to scan as a Hash or a JSON encoded String
+      # opts   - Options Hash
+      #   :index       - the name of the index to search
+      #   :type        - the document type to search
+      #   :scroll      - the keep alive time of the scrolling request (5 minutes by default)
+      #   :size        - the number of documents per shard to fetch per scroll
+      #   :search_type - set to 'scan' for scan query semantics
+      #
+      # Examples
+      #
+      #   scan = Scroller.new(client, {:search_type => 'scan', :query => {:match_all => {}}}, :index => 'test-1')
+      #   scan.each_document { |doc|
+      #     doc['_id']
+      #     doc['_source']
+      #   }
+      #
+      def initialize( client, query, opts = {} )
+        @client = client
+        @query  = query
+
+        @index       = opts.fetch(:index, nil)
+        @type        = opts.fetch(:type, nil)
+        @scroll      = opts.fetch(:scroll, '5m')
+        @size        = opts.fetch(:size, 50)
+        @search_type = opts.fetch(:search_type, nil)
+
+        @scroll_id = nil
+        @offset = 0
+      end
+
+      attr_reader :client, :query, :index, :type, :scroll, :size, :search_type, :scroll_id
+
+      # Iterate over all the search results from the scan query.
+      #
+      # block  - The block will be called for each set of matching documents
+      #          returned from executing the scan query.
+      #
+      # Yields a hits Hash containing the 'total' number of hits, current
+      # 'offset' into that total, and the Array of 'hits' document Hashes.
+      #
+      # Examples
+      #
+      #   scan.each do |hits|
+      #     hits['total']
+      #     hits['offset']
+      #     hits['hits'].each { |document| ... }
+      #   end
+      #
+      # Returns this Scan instance.
+      def each
+        loop do
+          body = do_scroll
+
+          hits = body['hits']
+          break if hits['hits'].empty?
+
+          hits['offset'] = @offset
+          @offset += hits['hits'].length
+
+          yield hits
+        end
+
+        self
+      end
+
+      # Iterate over each document from the scan query. This method is just a
+      # convenience wrapper around the `each` method; it iterates the Array of
+      # documents and passes them one by one to the block.
+      #
+      # block  - The block will be called for each document returned from
+      #          executing the scan query.
+      #
+      # Yields a document Hash.
+      #
+      # Examples
+      #
+      #   scan.each_document do |document|
+      #     document['_id']
+      #     document['_source']
+      #   end
+      #
+      # Returns this Scan instance.
+      def each_document( &block )
+        each { |hits| hits['hits'].each(&block) }
+      end
+
+      # Internal: Perform the actual scroll requests. This method wil call out
+      # to the `Client#start_scroll` and `Client#continue_scroll` methods while
+      # keeping track of the `scroll_id` internally.
+      #
+      # Returns the response body as a Hash.
+      def do_scroll
+        if scroll_id.nil?
+          body = client.start_scroll(scroll_opts)
+          if body['hits']['hits'].empty?
+            @scroll_id = body['_scroll_id']
+            return do_scroll
+          end
+        else
+          body = client.continue_scroll(scroll_id, scroll)
+        end
+
+        @scroll_id = body['_scroll_id']
+        body
+      end
+
+      # Internal: Returns the options Hash that should be passed to the initial
+      # `Client#start_scroll` method call.
+      def scroll_opts
+        hash = {
+          :scroll => scroll,
+          :size   => size,
+          :index  => index,
+          :type   => type,
+          :body   => query
+        }
+        hash[:search_type] = search_type unless search_type.nil?
+        hash
+      end
+
+    end  # Scroller
+  end  # Client
+end  # Elastomer

data/lib/elastomer/client/snapshot.rb

@@ -44,7 +44,7 @@ module Elastomer
           raise exception
         end
       end
-      alias :exist? :exists?
+      alias_method :exist?, :exists?
 
       # Create the snapshot.
       # See http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-snapshots.html#_snapshot

data/lib/elastomer/client/template.rb

@@ -27,7 +27,7 @@ module Elastomer
       def exists?
         client.cluster.templates.key? name
       end
-      alias :exist? :exists?
+      alias_method :exist?, :exists?
 
       # Get the template from the cluster.
       # See http://www.elasticsearch.org/guide/reference/api/admin-indices-templates/

data/lib/elastomer/client/warmer.rb

@@ -85,7 +85,7 @@ module Elastomer
           raise exception
         end
       end
-      alias :exist? :exists?
+      alias_method :exist?, :exists?
 
       # Internal: Returns a Hash containing default parameters.
       def defaults

data/lib/elastomer/notifications.rb

@@ -72,7 +72,7 @@ module Elastomer
     end
   end
 
-  # use ActiveSupport::Notifications as the default instrumentaiton service
+  # use ActiveSupport::Notifications as the default instrumentation service
   Notifications.service = ActiveSupport::Notifications
 
   # inject our instrument method into the Client class

data/lib/elastomer/version.rb

@@ -1,5 +1,5 @@
 module Elastomer
-  VERSION = '0.4.1'
+  VERSION = '0.5.0'
 
   def self.version
     VERSION

data/script/bootstrap

@@ -1,13 +1,6 @@
 #!/bin/sh
 set -e
 
-# force use of gcc under mac due to hiredis issues under clang
-if [ $(uname) = Darwin ]; then
-    export CC=/usr/bin/gcc
-    export CXX=/usr/bin/g++
-    export LD=/usr/bin/gcc
-fi
-
 cd "$(dirname "$0")/.."
 if bundle check 1>/dev/null 2>&1; then
     echo "Gem environment up-to-date"

data/script/cibuild

@@ -12,12 +12,17 @@ PATH="$(pwd)/bin:$(pwd)/script:/usr/share/rbenv/shims:$PATH"
 git log -n 1 || true
 echo
 
+# Create a snapshot dir and make it world writable
+export SNAPSHOT_DIR=/tmp/elastomer-client-snapshot-test
+mkdir -p $SNAPSHOT_DIR
+chmod a+rw $SNAPSHOT_DIR
+
 result=0
 
-export RBENV_VERSION="$(cat .ruby-version)"
-echo "Building under $RBENV_VERSION"
+export RBENV_VERSION="2.1.1-github"
+ruby -v
 script/bootstrap
-script/testsuite || result=$?
+script/test || result=$?
 
 if [ $result -ne 0 ]; then
     exit $result

data/script/test

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -e
+cd "$(dirname "$0")/.."
+
+fn=${1:-"test/**/*_test.rb"}
+bundle exec testrb -Itest $fn $@

data/test/client/bulk_test.rb

@@ -91,7 +91,7 @@ describe Elastomer::Client::Bulk do
     assert_bulk_create h['items'].last
 
     book_id = items.last['create']['_id']
-    assert_match %r/^\S{22}$/, book_id
+    assert_match %r/^\S{20,22}$/, book_id
 
     @index.refresh
 
@@ -112,7 +112,7 @@ describe Elastomer::Client::Bulk do
     assert_bulk_delete h['items'].last, 'expected to delete a book'
 
     book_id2 = items.first['create']['_id']
-    assert_match %r/^\S{22}$/, book_id2
+    assert_match %r/^\S{20,22}$/, book_id2
 
     @index.refresh
 

data/test/client/cluster_test.rb

@@ -43,6 +43,12 @@ describe Elastomer::Client::Cluster do
   end
 
   it 'gets the cluster settings' do
+    h = @cluster.get_settings
+    assert_instance_of Hash, h['persistent'], 'the persistent settings are returned'
+    assert_instance_of Hash, h['transient'], 'the transient settings are returned'
+  end
+
+  it 'gets the cluster settings with .settings' do
     h = @cluster.settings
     assert_instance_of Hash, h['persistent'], 'the persistent settings are returned'
     assert_instance_of Hash, h['transient'], 'the transient settings are returned'
@@ -105,9 +111,11 @@ describe Elastomer::Client::Cluster do
       @index.delete if @index.exists?
     end
 
-    it 'adds an alias' do
+    it 'adds and gets an alias' do
       hash = @cluster.get_aliases
-      assert_empty hash[@name]['aliases']
+      if es_version_always_returns_aliases?
+        assert_empty hash[@name]['aliases']
+      end
 
       @cluster.update_aliases \
         :add => {:index => @name, :alias => 'elastomer-test-unikitty'}
@@ -116,6 +124,19 @@ describe Elastomer::Client::Cluster do
       assert_equal ['elastomer-test-unikitty'], hash[@name]['aliases'].keys
     end
 
+    it 'adds and gets an alias with .aliases' do
+      hash = @cluster.aliases
+      if es_version_always_returns_aliases?
+        assert_empty hash[@name]['aliases']
+      end
+
+      @cluster.update_aliases \
+        :add => {:index => @name, :alias => 'elastomer-test-unikitty'}
+
+      hash = @cluster.aliases
+      assert_equal ['elastomer-test-unikitty'], hash[@name]['aliases'].keys
+    end
+
     it 'removes an alias' do
       @cluster.update_aliases \
         :add => {:index => @name, :alias => 'elastomer-test-unikitty'}

data/test/client/docs_test.rb

@@ -20,7 +20,7 @@ describe Elastomer::Client::Docs do
           :doc2 => {
             :_source => { :enabled => true }, :_all => { :enabled => false },
             :properties => {
-              :title  => { :type => 'string', :analyzer => 'standard' },
+              :title  => { :type => 'string', :analyzer => 'standard', :term_vector => 'with_positions_offsets' },
               :author => { :type => 'string', :index => 'not_analyzed' }
             }
           }
@@ -43,7 +43,7 @@ describe Elastomer::Client::Docs do
           :author => 'pea53'
 
     assert_created h
-    assert_match %r/^\S{22}$/, h['_id']
+    assert_match %r/^\S{20,22}$/, h['_id']
 
     h = @docs.index \
           :_id    => nil,
@@ -52,7 +52,7 @@ describe Elastomer::Client::Docs do
           :author => 'grantr'
 
     assert_created h
-    assert_match %r/^\S{22}$/, h['_id']
+    assert_match %r/^\S{20,22}$/, h['_id']
 
     h = @docs.index \
           :_id    => '',
@@ -61,7 +61,7 @@ describe Elastomer::Client::Docs do
           :author => 'mojombo'
 
     assert_created h
-    assert_match %r/^\S{22}$/, h['_id']
+    assert_match %r/^\S{20,22}$/, h['_id']
   end
 
   it 'uses the provided document ID' do
@@ -89,7 +89,29 @@ describe Elastomer::Client::Docs do
           :type => 'doc2'
 
     assert_created h
-    assert_match %r/^\S{22}$/, h['_id']
+    assert_match %r/^\S{20,22}$/, h['_id']
+  end
+
+  it 'extracts underscore attributes from the document' do
+    doc = {
+      :_id => '12',
+      :_type => 'doc2',
+      :_routing => 'author',
+      '_consistency' => 'all',
+      :title => "The Adventures of Huckleberry Finn",
+      :author => "Mark Twain",
+      :_unknown => "unknown attribute"
+    }
+
+    h = @docs.index doc
+    assert_created h
+    assert_equal '12', h['_id']
+
+    refute doc.key?(:_id)
+    refute doc.key?(:_type)
+    refute doc.key?(:_routing)
+    refute doc.key?('_consistency')
+    assert doc.key?(:_unknown)
   end
 
   it 'gets documents from the search index' do
@@ -103,6 +125,18 @@ describe Elastomer::Client::Docs do
     assert_equal 'mojombo', h['_source']['author']
   end
 
+  it 'checks if documents exist in the search index' do
+    refute @docs.exists?(:id => '1', :type => 'doc1')
+    populate!
+    assert @docs.exists?(:id => '1', :type => 'doc1')
+  end
+
+  it 'checks if documents exist in the search index with .exist?' do
+    refute @docs.exist?(:id => '1', :type => 'doc1')
+    populate!
+    assert @docs.exist?(:id => '1', :type => 'doc1')
+  end
+
   it 'gets multiple documents from the search index' do
     populate!
 
@@ -113,7 +147,7 @@ describe Elastomer::Client::Docs do
     authors = h['docs'].map { |d| d['_source']['author'] }
     assert_equal %w[mojombo pea53], authors
 
-    h = @docs.multi_get :ids => [2, 1], :_type => 'doc1'
+    h = @docs.multi_get({:ids => [2, 1]}, :type => 'doc1')
     authors = h['docs'].map { |d| d['_source']['author'] }
     assert_equal %w[defunkt mojombo], authors
 
@@ -124,6 +158,27 @@ describe Elastomer::Client::Docs do
     refute_found h['docs'][3]
   end
 
+  it 'gets multiple documents from the search index with .mget' do
+    populate!
+
+    h = @docs.mget :docs => [
+      { :_id => 1, :_type => 'doc1' },
+      { :_id => 1, :_type => 'doc2' }
+    ]
+    authors = h['docs'].map { |d| d['_source']['author'] }
+    assert_equal %w[mojombo pea53], authors
+
+    h = @docs.mget({:ids => [2, 1]}, :type => 'doc1')
+    authors = h['docs'].map { |d| d['_source']['author'] }
+    assert_equal %w[defunkt mojombo], authors
+
+    h = @index.docs('doc1').mget :ids => [1, 2, 3, 4]
+    assert_found h['docs'][0]
+    assert_found h['docs'][1]
+    refute_found h['docs'][2]
+    refute_found h['docs'][3]
+  end
+
   it 'deletes documents from the search index' do
     populate!
     @docs = @index.docs('doc2')
@@ -207,6 +262,16 @@ describe Elastomer::Client::Docs do
     assert_equal 'the author of resque', hit['_source']['title']
   end
 
+  it 'supports the shards search API' do
+    if es_version_supports_search_shards?
+      h = @docs.search_shards(:type => 'docs1')
+
+      assert h.key?("nodes"), "response contains \"nodes\" information"
+      assert h.key?("shards"), "response contains \"shards\" information"
+      assert h["shards"].is_a?(Array), "\"shards\" is an array"
+    end
+  end
+
   it 'counts documents' do
     h = @docs.count :q => '*:*'
     assert_equal 0, h['count']
@@ -375,6 +440,72 @@ describe Elastomer::Client::Docs do
     assert_equal 'mojombo', response['_source']['author']
   end
 
+  if es_version_1_x?
+    it 'provides access to term vector statistics' do
+      populate!
+
+      response = @docs.termvector :type => 'doc2', :id => 1, :fields => 'title'
+
+      assert response['term_vectors']['title']
+      assert response['term_vectors']['title']['field_statistics']
+      assert response['term_vectors']['title']['terms']
+      assert_equal %w[author logging of the], response['term_vectors']['title']['terms'].keys
+    end
+
+    it 'provides access to term vector statistics with .termvectors' do
+      populate!
+
+      response = @docs.termvectors :type => 'doc2', :id => 1, :fields => 'title'
+
+      assert response['term_vectors']['title']
+      assert response['term_vectors']['title']['field_statistics']
+      assert response['term_vectors']['title']['terms']
+      assert_equal %w[author logging of the], response['term_vectors']['title']['terms'].keys
+    end
+
+    it 'provides access to term vector statistics with .term_vector' do
+      populate!
+
+      response = @docs.term_vector :type => 'doc2', :id => 1, :fields => 'title'
+
+      assert response['term_vectors']['title']
+      assert response['term_vectors']['title']['field_statistics']
+      assert response['term_vectors']['title']['terms']
+      assert_equal %w[author logging of the], response['term_vectors']['title']['terms'].keys
+    end
+
+    it 'provides access to term vector statistics with .term_vectors' do
+      populate!
+
+      response = @docs.term_vectors :type => 'doc2', :id => 1, :fields => 'title'
+
+      assert response['term_vectors']['title']
+      assert response['term_vectors']['title']['field_statistics']
+      assert response['term_vectors']['title']['terms']
+      assert_equal %w[author logging of the], response['term_vectors']['title']['terms'].keys
+    end
+
+    it 'provides access to multi term vector statistics' do
+      populate!
+
+      response = @docs.multi_termvectors({:ids => [1, 2]}, :type => 'doc2', :fields => 'title', :term_statistics => true)
+      docs = response['docs']
+
+      assert docs
+      assert_equal(%w[1 2], docs.map { |h| h['_id'] }.sort)
+    end
+
+    it 'provides access to multi term vector statistics with .multi_term_vectors' do
+      populate!
+
+      response = @docs.multi_term_vectors({:ids => [1, 2]}, :type => 'doc2', :fields => 'title', :term_statistics => true)
+      docs = response['docs']
+
+      assert docs
+      assert_equal(%w[1 2], docs.map { |h| h['_id'] }.sort)
+    end
+  end
+
   # Create/index multiple documents.
   #
   # docs - An instance of Elastomer::Client::Docs or Elastomer::Client::Bulk. If