rubberband 0.0.2 → 0.0.4

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.4

data/lib/elasticsearch/client/hits.rb

@@ -3,11 +3,11 @@ require 'ostruct'
 module ElasticSearch
   module Api
     class Hit < OpenStruct
-      undef_method :id
+      undef_method :id if method_defined?(:id)
 
       def initialize(hit)
         hit = hit.dup
-        hit.merge!(hit.delete("_source"))
+        hit.merge!(hit["_source"]) if hit["_source"]
         hit["id"] ||= hit["_id"]
         super(hit)
         @table.freeze
@@ -44,7 +44,7 @@ module ElasticSearch
 
     class Hits
       include Pagination
-      attr_reader :hits, :total_entries, :_shards, :response, :facets
+      attr_reader :hits, :total_entries, :_shards, :response, :facets, :scroll_id
 
       def initialize(response, options={})
         @response = response
@@ -52,6 +52,7 @@ module ElasticSearch
         @total_entries = response["hits"]["total"]
         @_shards = response["_shards"]
         @facets = response["facets"]
+        @scroll_id = response["_scrollId"]
         populate(@options[:ids_only])
       end
 

data/lib/elasticsearch/client/index.rb

@@ -52,17 +52,18 @@ module ElasticSearch
       #from	 The starting from index of the hits to return. Defaults to 0.
       #size	 The number of hits to return. Defaults to 10.
       #search_type	 The type of the search operation to perform. Can be dfs_query_then_fetch, dfs_query_and_fetch, query_then_fetch, query_and_fetch. Defaults to query_then_fetch.
+      #scroll Get a scroll id to continue paging through the search results. Value is the time to keep a scroll request around, e.g. 5m
+      #ids_only Return ids instead of hits
       def search(query, options={})
         set_default_scope!(options)
 
         #TODO this doesn't work for facets, because they have a valid query key as element. need a list of valid toplevel keys in the search dsl
         #query = {:query => query} if query.is_a?(Hash) && !query[:query] # if there is no query element, wrap query in one
 
-        search_options = slice_hash(options, :df, :analyzer, :default_operator, :explain, :fields, :field, :sort, :from, :size, :search_type, :limit, :per_page, :page, :offset)
+        search_options = slice_hash(options, :df, :analyzer, :default_operator, :explain, :fields, :field, :sort, :from, :size, :search_type, :limit, :per_page, :page, :offset, :scroll)
 
-        search_options[:size] ||= search_options[:per_page] if search_options[:per_page]
-        search_options[:size] ||= search_options[:limit] if search_options[:limit]
-        search_options[:from] ||= search_options[:size] * (search_options[:page]-1) if search_options[:page] && search_options[:page] > 1
+        search_options[:size] ||= (search_options[:per_page] || search_options[:limit] || 10)
+        search_options[:from] ||= search_options[:size] * (search_options[:page].to_i-1) if search_options[:page] && search_options[:page].to_i > 1
         search_options[:from] ||= search_options[:offset] if search_options[:offset]
 
         search_options[:fields] = "_id" if options[:ids_only]
@@ -71,6 +72,12 @@ module ElasticSearch
         Hits.new(response, slice_hash(options, :per_page, :page, :ids_only)).freeze #ids_only returns array of ids instead of hits
       end
 
+      #ids_only Return ids instead of hits
+      def scroll(scroll_id, options={})
+        response = execute(:scroll, scroll_id)
+        Hits.new(response, slice_hash(options, :ids_only)).freeze
+      end 
+
       #df	 The default field to use when no field prefix is defined within the query.
       #analyzer	 The analyzer name to be used when analyzing the query string.
       #default_operator	 The default operator to be used, can be AND or OR. Defaults to OR.
@@ -82,7 +89,6 @@ module ElasticSearch
         response["count"].to_i #TODO check if count is nil
       end
 
-
       private
 
       def slice_hash(hash, *keys)

data/lib/elasticsearch/client/retrying_client.rb

@@ -1,9 +1,9 @@
 # mostly ripped from thrift_client
 
 module ElasticSearch
-  module RetryingClient
+  class NoServersAvailable < StandardError; end
 
-    class NoServersAvailable < StandardError; end
+  module RetryingClient
 
     RETRYING_DEFAULTS = {
       :randomize_server_list => true,
@@ -27,7 +27,7 @@ module ElasticSearch
     def connect!
       @current_server = next_server
       super
-    rescue ElasticSearch::Transport::RetryableError
+    rescue ElasticSearch::RetryableError
       retry
     end
 
@@ -42,6 +42,7 @@ module ElasticSearch
       @request_count = 0
     end
 
+    #TODO this can spin indefinitely if timeout > retry_period
     def next_server
       if @retry_period
         rebuild_live_server_list! if Time.now > @last_rebuild + @retry_period
@@ -66,7 +67,7 @@ module ElasticSearch
       @request_count += 1
       begin
         super
-      rescue ElasticSearch::Transport::RetryableError
+      rescue ElasticSearch::RetryableError
         disconnect!
         retry
       end

data/lib/elasticsearch/encoding.rb

@@ -1,2 +1,7 @@
 require 'encoding/base'
-require 'encoding/json'
+
+module ElasticSearch
+  module Encoding
+    autoload :JSON, 'encoding/json'
+  end
+end

data/lib/elasticsearch/transport.rb

@@ -1,3 +1,14 @@
 require "transport/base_protocol"
 require "transport/base"
-require "transport/http"
+
+module ElasticSearch
+  class ConnectionFailed < RetryableError; end
+  class HostResolutionError < RetryableError; end
+  class TimeoutError < RetryableError; end
+  class RequestError < FatalError; end
+
+  module Transport
+    autoload :HTTP, 'transport/http'
+    autoload :Thrift, 'transport/thrift'
+  end
+end

data/lib/elasticsearch/transport/base.rb

@@ -1,7 +1,8 @@
 module ElasticSearch
+  class RetryableError < StandardError; end
+  class FatalError < StandardError; end
+
   module Transport
-    class RetryableError < StandardError; end
-    class FatalError < StandardError; end
 
     DEFAULTS = {
       :encoder => ElasticSearch::Encoding::JSON
@@ -27,6 +28,10 @@ module ElasticSearch
       def encoder
         @encoder ||= @options[:encoder].new
       end
+
+      def request(method, operation, params={}, body=nil, headers={})
+        raise NotImplementedError
+      end
     end
   end
 end

data/lib/elasticsearch/transport/base_protocol.rb

@@ -1,98 +1,235 @@
 module ElasticSearch
   module Transport
-    module BaseProtocol
-      
-      # index api
-
+    module IndexProtocol
       def index(index, type, id, document, options={})
-        raise NotImplementedError
+        body = encoder.is_encoded?(document) ? document : encoder.encode(document)
+        if id.nil?
+          response = request(:post, {:index => index, :type => type}, {}, body)
+        else
+          response = request(:put, {:index => index, :type => type, :id => id}, {}, body)
+        end
+        handle_error(response) unless response.status == 200
+        encoder.decode(response.body)
       end
 
       def get(index, type, id, options={})
-        raise NotImplementedError
+        response = request(:get, {:index => index, :type => type, :id => id})
+        return nil if response.status == 404
+
+        handle_error(response) unless response.status == 200
+        hit = encoder.decode(response.body)
+        unescape_id!(hit) #TODO extract these two calls from here and search
+        set_encoding!(hit)
+        hit # { "_id", "_index", "_type", "_source" }
       end
 
-      def delete(options)
-        raise NotImplementedError
+      def delete(index, type, id, options={})
+        response = request(:delete,{:index => index, :type => type, :id => id})
+        handle_error(response) unless response.status == 200 # ElasticSearch always returns 200 on delete, even if the object doesn't exist
+        encoder.decode(response.body)
       end
 
       def search(index, type, query, options={})
-        raise NotImplementedError
+        if query.is_a?(Hash)
+          # patron cannot submit get requests with content, so if query is a hash, post it instead (assume a query hash is using the query dsl)
+          response = request(:post, {:index => index, :type => type, :op => "_search"}, options, encoder.encode(query))
+        else
+          response = request(:get, {:index => index, :type => type, :op => "_search"}, options.merge(:q => query))
+        end
+        handle_error(response) unless response.status == 200
+        results = encoder.decode(response.body)
+        # unescape ids
+        results["hits"]["hits"].each do |hit|
+          unescape_id!(hit)
+          set_encoding!(hit)
+        end
+        results # {"hits"=>{"hits"=>[{"_id", "_type", "_source", "_index", "_score"}], "total"}, "_shards"=>{"failed", "total", "successful"}}
+      end
+
+      def scroll(scroll_id)
+        response = request(:get, {:op => "_search/scroll"}, {:scroll_id => scroll_id })
+        handle_error(response) unless response.status == 200
+        results = encoder.decode(response.body)
+        # unescape ids
+        results["hits"]["hits"].each do |hit|
+          unescape_id!(hit)
+          set_encoding!(hit)
+        end
+        results # {"hits"=>{"hits"=>[{"_id", "_type", "_source", "_index", "_score"}], "total"}, "_shards"=>{"failed", "total", "successful"}, "_scrollId"}
       end
 
       def count(index, type, query, options={})
-        raise NotImplementedError
+        if query.is_a?(Hash)
+          # patron cannot submit get requests with content, so if query is a hash, post it instead (assume a query hash is using the query dsl)
+          response = request(:post, {:index => index, :type => type, :op => "_count"}, options, encoder.encode(query))
+        else
+          response = request(:get, {:index => index, :type => type, :op => "_count"}, options.merge(:q => query))
+        end
+        handle_error(response) unless response.status == 200
+        encoder.decode(response.body) # {"count", "_shards"=>{"failed", "total", "successful"}}
       end
+    end
 
-      # admin index api
+    module IndexAdminProtocol
       def index_status(index_list, options={})
-        raise NotImplementedError
+        standard_request(:get, {:index => index_list, :op => "_status"})
       end
 
       def create_index(index, create_options={}, options={})
-        raise NotImplementedError
+        standard_request(:put, {:index => index}, {}, encoder.encode(create_options))
       end
 
       def delete_index(index, options={})
-        raise NotImplementedError
+        standard_request(:delete, {:index => index})
       end
 
       def alias_index(operations, options={})
-        raise NotImplementedError
+        standard_request(:post, {:op => "_aliases"}, {}, encoder.encode(operations))
       end
 
       def update_mapping(index, type, mapping, options)
-        raise NotImplementedError
+        standard_request(:put, {:index => index, :type => type, :op => "_mapping"}, options, encoder.encode(mapping))
       end
 
       def flush(index_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => index_list, :op => "_flush"}, options, "")
       end
-
+      
       def refresh(index_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => index_list, :op => "_refresh"}, {}, "")
       end
-
+      
       def snapshot(index_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => index_list, :type => "_gateway", :op => "snapshot"}, {}, "")
       end
-
+      
       def optimize(index_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => index_list, :op => "_optimize"}, options, {})
       end
+    end
 
-
-      # admin cluster api
+    module ClusterAdminProtocol
       def cluster_health(index_list, options={})
-        raise NotImplementedError
+        standard_request(:get, {:index => "_cluster", :type => "health", :id => index_list}, options)
       end
 
       def cluster_state(options={})
-        raise NotImplementedError
+        standard_request(:get, {:index => "_cluster", :op => "state"})
       end
 
       def nodes_info(node_list, options={})
-        raise NotImplementedError
+        standard_request(:get, {:index => "_cluster", :type => "nodes", :id => node_list})
       end
 
       def nodes_stats(node_list, options={})
-        raise NotImplementedError
+        standard_request(:get, {:index => "_cluster", :type => "nodes", :id => node_list, :op => "stats"})
       end
 
       def shutdown_nodes(node_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => "_cluster", :type => "nodes", :id => node_list, :op => "_shutdown"}, options, "")
       end
 
       def restart_nodes(node_list, options={})
-        raise NotImplementedError
+        standard_request(:post, {:index => "_cluster", :type => "nodes", :id => node_list, :op =>  "_restart"}, options, "")
+      end
+    end
+
+    module ProtocolHelpers
+      private
+
+      def standard_request(*args)
+        response = request(*args)
+        handle_error(response) unless response.status == 200
+        encoder.decode(response.body)
       end
 
-      # misc
+      def handle_error(response)
+        raise RequestError, "(#{response.status}) #{response.body}"
+      end
+
+      # :index - one or many index names
+      # :type - one or many types
+      # :id - one id
+      # :op - optional operation
+      def generate_uri(options)
+        path = ""
+        path << "/#{Array(options[:index]).collect { |i| escape(i.downcase) }.join(",")}" if options[:index] && !options[:index].empty?
+        path << "/#{Array(options[:type]).collect { |t| escape(t) }.join(",")}" if options[:type] && !options[:type].empty?
+        path << "/#{Array(options[:id]).collect { |id| escape(id) }.join(",")}" if options[:id] && !options[:id].empty?
+        path << "/#{options[:op]}" if options[:op]
+        path
+      end
+
+      #doesn't handle arrays or hashes or what have you
+      def generate_query_string(params)
+        params.collect { |k,v| "#{escape(k.to_s)}=#{escape(v.to_s)}" }.join("&")
+      end
+
+      def unescape_id!(hit)
+        hit["_id"] = unescape(hit["_id"])
+        nil
+      end
+
+      def set_encoding!(hit)
+        encode_utf8(hit["_source"]) if hit["_source"].is_a?(String)
+        nil
+      end
+
+      # faster than CGI.escape
+      # stolen from RSolr, who stole it from Rack
+      def escape(string)
+        string.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+          #'%'+$1.unpack('H2'*$1.size).join('%').upcase
+          '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
+        }.tr(' ', '+')
+      end
+
+      def unescape(string)
+        CGI.unescape(string)
+      end
+
+      if ''.respond_to?(:force_encoding) && ''.respond_to?(:encoding)
+        # encodes the string as utf-8 in Ruby 1.9
+        def encode_utf8(string)
+          # ElasticSearch only ever returns json in UTF-8 (per the JSON spec) so we can use force_encoding here (#TODO what about ids? can we assume those are always ascii?)
+          string.force_encoding(::Encoding::UTF_8)
+        end
+      else
+        # returns the unaltered string in Ruby 1.8
+        def encode_utf8(string)
+          string
+        end
+      end
+
+      # Return the bytesize of String; uses String#size under Ruby 1.8 and
+      # String#bytesize under 1.9.
+      if ''.respond_to?(:bytesize)
+        def bytesize(string)
+          string.bytesize
+        end
+      else
+        def bytesize(string)
+          string.size
+        end
+      end
+    end
+
+    module BaseProtocol
+      include IndexProtocol
+      include IndexAdminProtocol
+      include ClusterAdminProtocol
+      include ProtocolHelpers
+
       def all_nodes
-        raise NotImplementedError
+        http_addresses = nodes_info([])["nodes"].collect { |id, node| node["http_address"] }
+        http_addresses.collect! do |a|
+          if a =~ /inet\[.*\/([\d.:]+)\]/
+            $1
+          end
+        end.compact!
+        http_addresses
       end
 
     end
   end
 end
-

data/lib/elasticsearch/transport/http.rb

@@ -5,11 +5,6 @@ module ElasticSearch
   module Transport
     class HTTP < Base
 
-      class ConnectionFailed < RetryableError; end
-      class HostResolutionError < RetryableError; end
-      class TimeoutError < RetryableError; end
-      class RequestError < FatalError; end
-
       DEFAULTS = {
         :timeout => 5
       }.freeze
@@ -24,220 +19,16 @@ module ElasticSearch
         @session.base_url = @server
         @session.timeout = @options[:timeout]
         @session.headers['User-Agent'] = 'ElasticSearch.rb v0.1'
-        request(:get, "/") # try a get to see if the server responds
-      end
-
-      # index api (TODO modulize)
-
-      def index(index, type, id, document, options={})
-        body = encoder.is_encoded?(document) ? document : encoder.encode(document)
-        if id.nil?
-          response = request(:post, generate_path(:index => index, :type => type), body)
-        else
-          response = request(:put, generate_path(:index => index, :type => type, :id => id), body)
-        end
-        handle_error(response) unless response.status == 200
-        encoder.decode(response.body)
-      end
-
-      def get(index, type, id, options={})
-        response = request(:get, generate_path(:index => index, :type => type, :id => id))
-        return nil if response.status == 404
-
-        handle_error(response) unless response.status == 200
-        hit = encoder.decode(response.body)
-        unescape_id!(hit) #TODO extract these two calls from here and search
-        set_encoding!(hit)
-        hit # { "_id", "_index", "_type", "_source" }
-      end
-
-      def delete(index, type, id, options={})
-        response = request(:delete, generate_path(:index => index, :type => type, :id => id))
-        handle_error(response) unless response.status == 200 # ElasticSearch always returns 200 on delete, even if the object doesn't exist
-        encoder.decode(response.body)
-      end
-
-      def search(index, type, query, options={})
-        if query.is_a?(Hash)
-          # patron cannot submit get requests with content, so if query is a hash, post it instead (assume a query hash is using the query dsl)
-          response = request(:post, generate_path(:index => index, :type => type, :op => "_search", :params => options), encoder.encode(query))
-        else
-          response = request(:get, generate_path(:index => index, :type => type, :op => "_search", :params => options.merge(:q => query)))
-        end
-        handle_error(response) unless response.status == 200
-        results = encoder.decode(response.body)
-        # unescape ids
-        results["hits"]["hits"].each do |hit|
-          unescape_id!(hit)
-          set_encoding!(hit)
-        end
-        results # {"hits"=>{"hits"=>[{"_id", "_type", "_source", "_index"}], "total"}, "_shards"=>{"failed", "total", "successful"}}
-      end
-
-      def count(index, type, query, options={})
-        if query.is_a?(Hash)
-          # patron cannot submit get requests with content, so if query is a hash, post it instead (assume a query hash is using the query dsl)
-          response = request(:post, generate_path(:index => index, :type => type, :op => "_count", :params => options), encoder.encode(query))
-        else
-          response = request(:get, generate_path(:index => index, :type => type, :op => "_count", :params => options.merge(:q => query)))
-        end
-        handle_error(response) unless response.status == 200
-        encoder.decode(response.body) # {"count", "_shards"=>{"failed", "total", "successful"}}
-      end
-
-      # admin index api (TODO modulize)
-      #
-      def index_status(index_list, options={})
-        standard_request(:get, generate_path(:index => index_list, :op => "_status"))
-      end
-
-      def create_index(index, create_options={}, options={})
-        standard_request(:put, generate_path(:index => index), encoder.encode(create_options))
-      end
-
-      def delete_index(index, options={})
-        standard_request(:delete, generate_path(:index => index))
-      end
-
-      def alias_index(operations, options={})
-        standard_request(:post, generate_path(:op => "_aliases"), encoder.encode(operations))
-      end
-
-      def update_mapping(index, type, mapping, options)
-        standard_request(:put, generate_path(:index => index, :type => type, :op => "_mapping", :params => options), encoder.encode(mapping))
-      end
-
-      def flush(index_list, options={})
-        standard_request(:post, generate_path(:index => index_list, :op => "_flush", :params => options), "")
-      end
-      
-      def refresh(index_list, options={})
-        standard_request(:post, generate_path(:index => index_list, :op => "_refresh"), "")
-      end
-      
-      def snapshot(index_list, options={})
-        standard_request(:post, generate_path(:index => index_list, :type => "_gateway", :op => "snapshot"), "")
-      end
-      
-      def optimize(index_list, options={})
-        standard_request(:post, generate_path(:index => index_list, :op => "_optimize", :params => options), "")
-      end
-
-      # admin cluster api (TODO modulize)
-      
-      def cluster_health(index_list, options={})
-        standard_request(:get, generate_path(:index => "_cluster", :type => "health", :id => index_list, :params => options))
-      end
-
-      def cluster_state(options={})
-        standard_request(:get, generate_path(:index => "_cluster", :op => "state"))
-      end
-
-      def nodes_info(node_list, options={})
-        standard_request(:get, generate_path(:index => "_cluster", :type => "nodes", :id => node_list))
-      end
-
-      def nodes_stats(node_list, options={})
-        standard_request(:get, generate_path(:index => "_cluster", :type => "nodes", :id => node_list, :op => "stats"))
-      end
-
-      def shutdown_nodes(node_list, options={})
-        standard_request(:post, generate_path(:index => "_cluster", :type => "nodes", :id => node_list, :op => "_shutdown", :params => options), "")
-      end
-
-      def restart_nodes(node_list, options={})
-        standard_request(:post, generate_path(:index => "_cluster", :type => "nodes", :id => node_list, :op =>  "_restart", :params => options), "")
-      end
-
-      # misc helper methods (TODO modulize)
-      def all_nodes
-        http_addresses = nodes_info([])["nodes"].collect { |id, node| node["http_address"] }
-        http_addresses.collect! do |a|
-          if a =~ /inet\[.*\/([\d.:]+)\]/
-            $1
-          end
-        end.compact!
-        http_addresses
       end
 
       private
 
-      def standard_request(*args)
-        response = request(*args)
-        handle_error(response) unless response.status == 200
-        encoder.decode(response.body)
-      end
-
-      # :index - one or many index names
-      # :type - one or many types
-      # :id - one id
-      # :params - hash of query params
-      def generate_path(options)
-        path = ""
-        path << "/#{Array(options[:index]).collect { |i| escape(i.downcase) }.join(",")}" if options[:index] && !options[:index].empty?
-        path << "/#{Array(options[:type]).collect { |t| escape(t) }.join(",")}" if options[:type] && !options[:type].empty?
-        path << "/#{Array(options[:id]).collect { |id| escape(id) }.join(",")}" if options[:id] && !options[:id].empty?
-        path << "/#{options[:op]}" if options[:op]
-        path << "?" << query_string(options[:params]) if options[:params] && !options[:params].empty?
-        path
-      end
-
-      def unescape_id!(hit)
-        hit["_id"] = unescape(hit["_id"])
-        nil
-      end
-
-      def set_encoding!(hit)
-        encode_utf8(hit["_source"])
-        nil
-      end
-
-      # faster than CGI.escape
-      # stolen from RSolr, which stole it from Rack
-      def escape(string)
-        string.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
-          #'%'+$1.unpack('H2'*$1.size).join('%').upcase
-          '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
-        }.tr(' ', '+')
-      end
-
-      def unescape(string)
-        CGI.unescape(string)
-      end
-
-      if ''.respond_to?(:force_encoding) && ''.respond_to?(:encoding)
-        # encodes the string as utf-8 in Ruby 1.9
-        def encode_utf8(string)
-          # ElasticSearch only ever returns json in UTF-8 (per the JSON spec) so we can use force_encoding here (#TODO what about ids? can we assume those are always ascii?)
-          string.force_encoding(Encoding::UTF_8)
-        end
-      else
-        # returns the unaltered string in Ruby 1.8
-        def encode_utf8(string)
-          string
-        end
-      end
-
-      # Return the bytesize of String; uses String#size under Ruby 1.8 and
-      # String#bytesize under 1.9.
-      if ''.respond_to?(:bytesize)
-        def bytesize(string)
-          string.bytesize
-        end
-      else
-        def bytesize(string)
-          string.size
-        end
-      end
-
-      #doesn't handle arrays or hashes or what have you
-      def query_string(params)
-        params.collect { |k,v| "#{escape(k.to_s)}=#{escape(v.to_s)}" }.join("&")
-      end
-
-      def request(method, path, body=nil, headers={})
+      def request(method, operation, params={}, body=nil, headers={})
         begin
-          #puts "request: #{@session.base_url}#{path} body:#{body}"
+          uri = generate_uri(operation)
+          query = generate_query_string(params)
+          path = [uri, query].join("?")
+          #puts "request: #{@server} #{path} #{body}"
           response = @session.request(method, path, headers, :data => body)
           handle_error(response) if response.status >= 500
           response
@@ -247,17 +38,13 @@ module ElasticSearch
             raise ConnectionFailed
           when Patron::HostResolutionError
             raise HostResolutionError
-          when TimeoutError
+          when Patron::TimeoutError
             raise TimeoutError
           else
             raise e
           end
         end
       end
-
-      def handle_error(response)
-        raise RequestError, "(#{response.status}) #{response.body}"
-      end
     end
   end
 end