tire-erez 0.5.4

data/lib/tire/dsl.rb

@@ -0,0 +1,114 @@
+module Tire
+  module DSL
+
+    def configure(&block)
+      Configuration.class_eval(&block)
+    end
+
+    def search(indices=nil, options={}, &block)
+      if block_given?
+        Search::Search.new(indices, options, &block)
+      else
+        payload = case options
+          when Hash    then
+            options
+          when String  then
+            Tire.warn "Passing the payload as a JSON string in Tire.search has been deprecated, " +
+                       "please use the block syntax or pass a plain Hash."
+            options
+          else raise ArgumentError, "Please pass a Ruby Hash or String with JSON"
+        end
+        unless options.empty?
+          Search::Search.new(indices, :payload => payload)
+        else
+          Search::Search.new(indices)
+        end
+      end
+    rescue Exception => error
+      STDERR.puts "[REQUEST FAILED] #{error.class} #{error.message rescue nil}\n"
+      raise
+    ensure
+    end
+
+    # Build and perform a [multi-search](http://elasticsearch.org/guide/reference/api/multi-search.html)
+    # request.
+    #
+    #     s = Tire.multi_search 'clients' do
+    #           search :names do
+    #             query { match :name, 'carpenter' }
+    #           end
+    #           search :counts, search_type: 'count' do
+    #             query { match [:name, :street, :occupation], 'carpenter' }
+    #           end
+    #           search :vip, index: 'clients-vip' do
+    #             query { string "last_name:carpenter" }
+    #           end
+    #           search() { query {all} }
+    #         end
+    #
+    # The DSL allows you to perform multiple searches and get corresponding results
+    # in a single HTTP request, saving network roundtrips.
+    #
+    # Use the `search` method in the block to define a search request with the
+    # regular Tire's DSL (`query`, `facet`, etc).
+    #
+    # You can pass options such as `search_type`, `routing`, etc.,
+    # as well as a different `index` and/or `type` to individual searches.
+    #
+    # You can give single searches names, to be able to refer to them later.
+    #
+    # The results are returned as an enumerable collection of {Tire::Results::Collection} instances.
+    #
+    # You may simply iterate over them with `each`:
+    #
+    #     s.results.each do |results|
+    #       puts results.map(&:name)
+    #     end
+    #
+    # To iterate over named results, use the `each_pair` method:
+    #
+    #     s.results.each_pair do |name,results|
+    #       puts "Search #{name} got #{results.size} results"
+    #     end
+    #
+    # You can get a specific named result:
+    #
+    #     search.results[:vip]
+    #
+    # You can mix & match named and non-named searches in the definition; the non-named
+    # searches will be zero-based numbered, so you can refer to them:
+    #
+    #     search.results[3] # Results for the last query
+    #
+    # To log the multi-search request, use the standard `to_curl` method (or set up a logger):
+    #
+    #     print search.to_curl
+    #
+    def multi_search(indices=nil, options={}, &block)
+      Search::Multi::Search.new(indices, options, &block)
+    rescue Exception => error
+      STDERR.puts "[REQUEST FAILED] #{error.class} #{error.message rescue nil}\n"
+      raise
+    ensure
+    end
+    alias :multisearch :multi_search
+    alias :msearch     :multi_search
+
+    def count(indices=nil, options={}, &block)
+      Search::Count.new(indices, options, &block).value
+    end
+
+    def index(name, &block)
+      Index.new(name, &block)
+    end
+
+    def scan(names, options={}, &block)
+      Search::Scan.new(names, options, &block)
+    end
+
+    def aliases
+      Alias.all
+    end
+
+  end
+end

data/lib/tire/http/client.rb

@@ -0,0 +1,62 @@
+module Tire
+
+  module HTTP
+
+    module Client
+
+      class RestClient
+        ConnectionExceptions = [::RestClient::ServerBrokeConnection, ::RestClient::RequestTimeout]
+
+        def self.get(url, data=nil)
+          perform ::RestClient::Request.new(:method => :get, :url => url, :payload => data).execute
+        rescue *ConnectionExceptions
+          raise
+        rescue ::RestClient::Exception => e
+          Response.new e.http_body, e.http_code
+        end
+
+        def self.post(url, data)
+          perform ::RestClient.post(url, data)
+        rescue *ConnectionExceptions
+          raise
+        rescue ::RestClient::Exception => e
+          Response.new e.http_body, e.http_code
+        end
+
+        def self.put(url, data)
+          perform ::RestClient.put(url, data)
+        rescue *ConnectionExceptions
+          raise
+        rescue ::RestClient::Exception => e
+          Response.new e.http_body, e.http_code
+        end
+
+        def self.delete(url)
+          perform ::RestClient.delete(url)
+        rescue *ConnectionExceptions
+          raise
+        rescue ::RestClient::Exception => e
+          Response.new e.http_body, e.http_code
+        end
+
+        def self.head(url)
+          perform ::RestClient.head(url)
+        rescue *ConnectionExceptions
+          raise
+        rescue ::RestClient::Exception => e
+          Response.new e.http_body, e.http_code
+        end
+
+        private
+
+        def self.perform(response)
+          Response.new response.body, response.code, response.headers
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/tire/http/clients/curb.rb

@@ -0,0 +1,61 @@
+require 'curb'
+
+module Tire
+
+  module HTTP
+
+    module Client
+
+      class Curb
+        @client = ::Curl::Easy.new
+        @client.resolve_mode = :ipv4
+
+        # @client.verbose = true
+
+        def self.get(url, data=nil)
+          @client.url = url
+
+          # FIXME: Curb cannot post bodies with GET requests?
+          #        Roy Fielding seems to approve:
+          #        <http://tech.groups.yahoo.com/group/rest-discuss/message/9962>
+          if data
+            @client.post_body = data
+            @client.http_post
+          else
+            @client.http_get
+          end
+          Response.new @client.body_str, @client.response_code
+        end
+
+        def self.post(url, data)
+          @client.url = url
+          @client.post_body = data
+          @client.http_post
+          Response.new @client.body_str, @client.response_code
+        end
+
+        def self.put(url, data)
+          @client.url = url
+          @client.http_put data
+          Response.new @client.body_str, @client.response_code
+        end
+
+        def self.delete(url)
+          @client.url = url
+          @client.http_delete
+          Response.new @client.body_str, @client.response_code
+        end
+
+        def self.head(url)
+          @client.url = url
+          @client.http_head
+          Response.new @client.body_str, @client.response_code
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/tire/http/clients/faraday.rb

@@ -0,0 +1,71 @@
+require 'faraday'
+
+# A Faraday-based HTTP client, which allows you to choose a HTTP client.
+#
+# See <https://github.com/technoweenie/faraday/tree/master/lib/faraday/adapter>
+#
+# NOTE: Tire will switch to Faraday for the HTTP abstraction layer. This client is a temporary solution.
+#
+# Example:
+# --------
+#
+#     require 'typhoeus'
+#     require 'tire/http/clients/faraday'
+#
+#     Tire.configure do |config|
+#     
+#       # Unless specified, tire will use Faraday.default_adapter and no middleware
+#       Tire::HTTP::Client::Faraday.faraday_middleware = Proc.new do |builder|
+#         builder.adapter :typhoeus
+#       end
+#     
+#       config.client(Tire::HTTP::Client::Faraday)
+#     
+#     end
+#
+#
+module Tire
+  module HTTP
+    module Client
+      class Faraday
+
+        # Default middleware stack.
+        DEFAULT_MIDDLEWARE = Proc.new do |builder|
+          builder.adapter ::Faraday.default_adapter
+        end
+
+        class << self
+          # A customized stack of Faraday middleware that will be used to make each request.
+          attr_accessor :faraday_middleware
+
+          def get(url, data = nil)
+            request(:get, url, data)
+          end
+
+          def post(url, data)
+            request(:post, url, data)
+          end
+
+          def put(url, data)
+            request(:put, url, data)
+          end
+
+          def delete(url, data = nil)
+            request(:delete, url, data)
+          end
+
+          def head(url)
+            request(:head, url)
+          end
+
+          private
+          def request(method, url, data = nil)
+            conn = ::Faraday.new( &(faraday_middleware || DEFAULT_MIDDLEWARE) )
+            response = conn.run_request(method, url, data, nil)
+            Response.new(response.body, response.status, response.headers)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/tire/http/response.rb

@@ -0,0 +1,27 @@
+module Tire
+
+  module HTTP
+
+    class Response
+      attr_reader :body, :code, :headers
+
+      def initialize(body, code, headers={})
+        @body, @code, @headers = body, code.to_i, headers
+      end
+
+      def success?
+        code > 0 && code < 400
+      end
+
+      def failure?
+        ! success?
+      end
+
+      def to_s
+        [code, body].join(' : ')
+      end
+    end
+
+  end
+
+end

data/lib/tire/index.rb

@@ -0,0 +1,443 @@
+module Tire
+  class Index
+
+    attr_reader :name, :response
+
+    def initialize(name, &block)
+      @name = name
+      block.arity < 1 ? instance_eval(&block) : block.call(self) if block_given?
+    end
+
+    def url
+      "#{Configuration.url}/#{@name}"
+    end
+
+    def exists?
+      @response = Configuration.client.head("#{url}")
+      @response.success?
+
+    ensure
+      curl = %Q|curl -I "#{url}"|
+      logged('HEAD', curl)
+    end
+
+    def delete
+      @response = Configuration.client.delete url
+      @response.success?
+
+    ensure
+      curl = %Q|curl -X DELETE #{url}|
+      logged('DELETE', curl)
+    end
+
+    def create(options={})
+      @options = options
+      @response = Configuration.client.post url, MultiJson.encode(options)
+      @response.success? ? @response : false
+
+    ensure
+      curl = %Q|curl -X POST #{url} -d '#{MultiJson.encode(options, :pretty => Configuration.pretty)}'|
+      logged('CREATE', curl)
+    end
+
+    def add_alias(alias_name, configuration={})
+      Alias.create(configuration.merge( :name => alias_name, :index => @name ) )
+    end
+
+    def remove_alias(alias_name)
+      Alias.find(alias_name) { |a| a.indices.delete @name }.save
+    end
+
+    def aliases(alias_name=nil)
+      alias_name ? Alias.all(@name).select { |a| a.name == alias_name }.first : Alias.all(@name)
+    end
+
+    def mapping
+      @response = Configuration.client.get("#{url}/_mapping")
+      MultiJson.decode(@response.body)[@name]
+    end
+
+    def settings
+      @response = Configuration.client.get("#{url}/_settings")
+      MultiJson.decode(@response.body)[@name]['settings']
+    end
+
+    def store(*args)
+      document, options = args
+
+      id       = get_id_from_document(document)
+      type     = get_type_from_document(document)
+      document = convert_document_to_json(document)
+
+      options ||= {}
+      params    = {}
+
+      if options[:percolate]
+        params[:percolate] = options[:percolate]
+        params[:percolate] = "*" if params[:percolate] === true
+      end
+
+      params[:parent]  = options[:parent]  if options[:parent]
+      params[:routing] = options[:routing] if options[:routing]
+      params[:replication] = options[:replication] if options[:replication]
+
+      params_encoded = params.empty? ? '' : "?#{params.to_param}"
+
+      url  = id ? "#{self.url}/#{type}/#{id}#{params_encoded}" : "#{self.url}/#{type}/#{params_encoded}"
+
+      @response = Configuration.client.post url, document
+      MultiJson.decode(@response.body)
+    ensure
+      curl = %Q|curl -X POST "#{url}" -d '#{document}'|
+      logged([type, id].join('/'), curl)
+    end
+
+    # Performs a [bulk](http://www.elasticsearch.org/guide/reference/api/bulk.html) request
+    #
+    #     @myindex.bulk :index, [ {id: 1, title: 'One'}, { id: 2, title: 'Two', _version: 3 } ], refresh: true
+    #
+    # Pass the action (`index`, `create`, `delete`) as the first argument, the collection of documents as
+    # the second argument, and URL parameters as the last option.
+    #
+    # Any _meta_ information contained in documents (such as `_routing` or `_parent`) is extracted
+    # and added to the "header" line.
+    #
+    # Shortcut methods `bulk_store`, `bulk_delete` and `bulk_create` are available.
+    #
+    def bulk(action, documents, options={})
+      return false if documents.empty?
+
+      # TODO: A more Ruby-like DSL notation should be supported:
+      #
+      #     Tire.index('myindex').bulk do
+      #       create id: 1, title: 'bar', _routing: 'abc'
+      #       delete id: 1
+      #       # ...
+      #     end
+
+      payload = documents.map do |document|
+        type = get_type_from_document(document, :escape => false) # Do not URL-escape the _type
+        id   = get_id_from_document(document)
+
+        if ENV['DEBUG']
+          STDERR.puts "[ERROR] Document #{document.inspect} does not have ID" unless id
+        end
+
+        header = { action.to_sym => { :_index => name, :_type => type, :_id => id } }
+
+        if document.respond_to?(:to_hash) && hash = document.to_hash
+          meta = {}
+          meta[:_version]   = hash.delete(:_version)
+          meta[:_routing]   = hash.delete(:_routing)
+          meta[:_percolate] = hash.delete(:_percolate)
+          meta[:_parent]    = hash.delete(:_parent)
+          meta[:_timestamp] = hash.delete(:_timestamp)
+          meta[:_ttl]       = hash.delete(:_ttl)
+          meta              = meta.reject { |name,value| !value || value.empty? }
+          header[action.to_sym].update(meta)
+        end
+
+        output = []
+        output << MultiJson.encode(header)
+        output << convert_document_to_json(document) unless action.to_s == 'delete'
+        output.join("\n")
+      end
+      payload << ""
+
+      tries = 5
+      count = 0
+
+      begin
+        params = {}
+        params[:consistency] = options.delete(:consistency)
+        params[:refresh]     = options.delete(:refresh)
+        params               = params.reject { |name,value| !value }
+        params_encoded       = params.empty? ? '' : "?#{params.to_param}"
+
+        @response = Configuration.client.post("#{url}/_bulk#{params_encoded}", payload.join("\n"))
+        raise RuntimeError, "#{@response.code} > #{@response.body}" if @response && @response.failure?
+        @response
+      rescue StandardError => error
+        if count < tries
+          count += 1
+          STDERR.puts "[ERROR] #{error.message}, retrying (#{count})..."
+          retry
+        else
+          STDERR.puts "[ERROR] Too many exceptions occured, giving up. The HTTP response was: #{error.message}"
+          raise if options[:raise]
+        end
+
+      ensure
+        curl = %Q|curl -X POST "#{url}/_bulk" --data-binary '{... data omitted ...}'|
+        logged('_bulk', curl)
+      end
+
+    end
+
+    def bulk_create(documents, options={})
+      bulk :create, documents, options
+    end
+
+    def bulk_store(documents, options={})
+      bulk :index, documents, options
+    end
+
+    def bulk_delete(documents, options={})
+      bulk :delete, documents, options
+    end
+
+    def import(klass_or_collection, options={})
+      case
+        when method = options.delete(:method)
+          options = {:page => 1, :per_page => 1000}.merge options
+          while documents = klass_or_collection.send(method.to_sym, options.merge(:page => options[:page])) \
+                            and documents.to_a.length > 0
+
+            documents = yield documents if block_given?
+
+            bulk_store documents, options
+            options[:page] += 1
+          end
+
+        when klass_or_collection.respond_to?(:map)
+          documents = block_given? ? yield(klass_or_collection) : klass_or_collection
+          bulk_store documents, options
+
+        else
+          raise ArgumentError, "Please pass either an Enumerable compatible class, or a collection object" +
+                               "with a method for fetching records in batches (such as 'paginate')"
+      end
+    end
+
+    def reindex(name, options={}, &block)
+      new_index = Index.new(name)
+      new_index.create(options) unless new_index.exists?
+
+      transform = options.delete(:transform)
+
+      Search::Scan.new(self.name, &block).each do |results|
+
+        documents = results.map do |document|
+          document  = document.to_hash.except(:type, :_index, :_explanation, :_score, :_version, :highlight, :sort)
+          document  = transform.call(document) if transform
+          document
+        end
+
+        new_index.bulk_store documents
+      end
+    end
+
+    def remove(*args)
+      if args.size > 1
+        type, document = args
+        type           = Utils.escape(type)
+        id             = get_id_from_document(document) || document
+      else
+        document = args.pop
+        type     = get_type_from_document(document)
+        id       = get_id_from_document(document) || document
+      end
+      raise ArgumentError, "Please pass a document ID" unless id
+
+      url    = "#{self.url}/#{type}/#{id}"
+      result = Configuration.client.delete url
+      MultiJson.decode(result.body) if result.success?
+
+    ensure
+      curl = %Q|curl -X DELETE "#{url}"|
+      logged(id, curl)
+    end
+
+    def retrieve(type, id, options={})
+      raise ArgumentError, "Please pass a document ID" unless id
+
+      type      = Utils.escape(type)
+      url       = "#{self.url}/#{type}/#{id}"
+
+      params    = {}
+      params[:routing]    = options[:routing] if options[:routing]
+      params[:fields]     = options[:fields]  if options[:fields]
+      params[:preference] = options[:preference] if options[:preference]
+      params_encoded      = params.empty? ? '' : "?#{params.to_param}"
+
+      @response = Configuration.client.get "#{url}#{params_encoded}"
+
+      h = MultiJson.decode(@response.body)
+      wrapper = options[:wrapper] || Configuration.wrapper
+      if wrapper == Hash then h
+      else
+        return nil if h['exists'] == false
+        document = h['_source'] || h['fields'] || {}
+        document.update('id' => h['_id'], '_type' => h['_type'], '_index' => h['_index'], '_version' => h['_version'])
+        wrapper.new(document)
+      end
+
+    ensure
+      curl = %Q|curl -X GET "#{url}"|
+      logged(id, curl)
+    end
+
+    def update(type, id, payload={}, options={})
+      raise ArgumentError, "Please pass a document type" unless type
+      raise ArgumentError, "Please pass a document ID"   unless id
+      raise ArgumentError, "Please pass a script or partial document in the payload hash" unless payload[:script] || payload[:doc]
+
+      type      = Utils.escape(type)
+      url       = "#{self.url}/#{type}/#{id}/_update"
+      url      += "?#{options.to_param}" unless options.keys.empty?
+      @response = Configuration.client.post url, MultiJson.encode(payload)
+      MultiJson.decode(@response.body)
+
+    ensure
+      curl = %Q|curl -X POST "#{url}" -d '#{MultiJson.encode(payload, :pretty => Configuration.pretty)}'|
+      logged(id, curl)
+    end
+
+    def refresh
+      @response = Configuration.client.post "#{url}/_refresh", ''
+
+    ensure
+      curl = %Q|curl -X POST "#{url}/_refresh"|
+      logged('_refresh', curl)
+    end
+
+    def open(options={})
+      # TODO: Remove the duplication in the execute > rescue > ensure chain
+      @response = Configuration.client.post "#{url}/_open", MultiJson.encode(options)
+      MultiJson.decode(@response.body)['ok']
+
+    ensure
+      curl = %Q|curl -X POST "#{url}/_open"|
+      logged('_open', curl)
+    end
+
+    def close(options={})
+      @response = Configuration.client.post "#{url}/_close", MultiJson.encode(options)
+      MultiJson.decode(@response.body)['ok']
+
+    ensure
+      curl = %Q|curl -X POST "#{url}/_close"|
+      logged('_close', curl)
+    end
+
+    def analyze(text, options={})
+      options = {:pretty => true}.update(options)
+      params  = options.to_param
+      @response = Configuration.client.get "#{url}/_analyze?#{params}", text
+      @response.success? ? MultiJson.decode(@response.body) : false
+
+    ensure
+      curl = %Q|curl -X GET "#{url}/_analyze?#{params}" -d '#{text}'|
+      logged('_analyze', curl)
+    end
+
+    def register_percolator_query(name, options={}, &block)
+      options[:query] = Search::Query.new(&block).to_hash if block_given?
+
+      @response = Configuration.client.put "#{Configuration.url}/_percolator/#{@name}/#{name}", MultiJson.encode(options)
+      MultiJson.decode(@response.body)['ok']
+
+    ensure
+      curl = %Q|curl -X PUT "#{Configuration.url}/_percolator/#{@name}/#{name}?pretty" -d '#{MultiJson.encode(options, :pretty => Configuration.pretty)}'|
+      logged('_percolator', curl)
+    end
+
+    def unregister_percolator_query(name)
+      @response = Configuration.client.delete "#{Configuration.url}/_percolator/#{@name}/#{name}"
+      MultiJson.decode(@response.body)['ok']
+
+    ensure
+      curl = %Q|curl -X DELETE "#{Configuration.url}/_percolator/#{@name}"|
+      logged('_percolator', curl)
+    end
+
+    def percolate(*args, &block)
+      document = args.shift
+      type     = get_type_from_document(document)
+
+      document = MultiJson.decode convert_document_to_json(document)
+
+      query = Search::Query.new(&block).to_hash if block_given?
+
+      payload = { :doc => document }
+      payload.update( :query => query ) if query
+
+      @response = Configuration.client.get "#{url}/#{type}/_percolate", MultiJson.encode(payload)
+      MultiJson.decode(@response.body)['tire_matches']
+
+    ensure
+      curl = %Q|curl -X GET "#{url}/#{type}/_percolate?pretty" -d '#{MultiJson.encode(payload, :pretty => Configuration.pretty)}'|
+      logged('_percolate', curl)
+    end
+
+    def logged(endpoint='/', curl='')
+      if Configuration.logger
+        error = $!
+
+        Configuration.logger.log_request endpoint, @name, curl
+
+        code = @response ? @response.code : error.class rescue 'N/A'
+
+        if Configuration.logger.level.to_s == 'debug'
+          body = if @response && @response.body && !@response.body.to_s.empty?
+              MultiJson.encode( MultiJson.load(@response.body), :pretty => Configuration.pretty)
+            elsif error && error.message && !error.message.to_s.empty?
+              MultiJson.encode( MultiJson.load(error.message), :pretty => Configuration.pretty) rescue ''
+            else ''
+          end
+        else
+          body = ''
+        end
+
+        Configuration.logger.log_response code, nil, body
+      end
+    end
+
+    def get_type_from_document(document, options={})
+      options = {:escape => true}.merge(options)
+
+      old_verbose, $VERBOSE = $VERBOSE, nil # Silence Object#type deprecation warnings
+      type = case
+        when document.respond_to?(:document_type)
+          document.document_type
+        when document.is_a?(Hash)
+          document[:_type] || document['_type'] || document[:type] || document['type']
+        when document.respond_to?(:_type)
+          document._type
+        when document.respond_to?(:type) && document.type != document.class
+          document.type
+        end
+      $VERBOSE = old_verbose
+
+      type ||= 'document'
+      options[:escape] ? Utils.escape(type) : type
+    end
+
+    def get_id_from_document(document)
+      old_verbose, $VERBOSE = $VERBOSE, nil # Silence Object#id deprecation warnings
+      id = case
+        when document.is_a?(Hash)
+          document[:_id] || document['_id'] || document[:id] || document['id']
+        when document.respond_to?(:id) && document.id != document.object_id
+          document.id.as_json
+      end
+      $VERBOSE = old_verbose
+      id
+    end
+
+    def convert_document_to_json(document)
+      document = case
+        when document.is_a?(String)
+          if ENV['DEBUG']
+            Tire.warn "Passing the document as JSON string has been deprecated, " +
+                       "please pass an object which responds to `to_indexed_json` or a plain Hash."
+          end
+          document
+        when document.respond_to?(:to_indexed_json) then document.to_indexed_json
+        else raise ArgumentError, "Please pass a JSON string or object with a 'to_indexed_json' method," +
+                                  "'#{document.class}' given."
+      end
+    end
+
+  end
+end