stretcher 1.13.0 → 1.14.0

data/README.md

@@ -16,6 +16,7 @@ A concise, fast ElasticSearch client designed to reflect the actual elastic sear
 * Configurable logging
 * Pure, threadsafe, ruby
 * Easily swap HTTP clients via Faraday
+* Tested against Ruby 2.0,1.9,1.8.7, Jruby, and Rubinius
 
 ## Installation
 

data/lib/stretcher/es_component.rb

@@ -17,7 +17,6 @@ module Stretcher
         body = generic_opts
       end
 
-      logger.info { "Stretcher Search: curl -XGET '#{Util.qurl(path_uri('_search'), query_opts)}' -d '#{body.to_json}'" }
       response = request(:get, "_search", query_opts) do |req|
         req.body = body
       end
@@ -28,10 +27,10 @@ module Stretcher
       request(:post, "_refresh")
     end
 
-    def request(method, path=nil, query_opts=nil, *args, &block)
+    def request(method, path=nil, params={}, body=nil, headers={}, &block)
       prefixed_path = path_uri(path)
       raise "Cannot issue request, no server specified!" unless @server
-      @server.request(method, prefixed_path, query_opts, *args, &block)
+      @server.request(method, prefixed_path, params, body, headers, &block)
     end
 
     def do_delete_query(query)

data/lib/stretcher/index.rb

@@ -26,21 +26,39 @@ module Stretcher
     #
     #    docs = [{"_type" => "tweet", "_id" => 91011, "text" => "Bulked"}]
     #    server.index(:foo).bulk_index(docs)
-    def bulk_index(documents)
-      @server.bulk documents.reduce("") {|post_data, d_raw|
+    def bulk_index(documents, options={})
+      bulk_action(:index, documents, options)
+    end
+
+    # Given a hash of documents, will bulk delete
+    #
+    #    docs = [{"_type" => "tweet", "_id" => 91011}]
+    #    server.index(:foo).bulk_delete(docs)
+    def bulk_delete(documents, options={})
+      bulk_action(:delete, documents, options)
+    end
+
+    def bulk_action(action, documents, options={})
+      action=action.to_sym
+      
+      body = documents.reduce("") {|post_data, d_raw|
         d = Hashie::Mash.new(d_raw)
-        action_meta = {"index" => {"_index" => name, "_type" => d["_type"], "_id" => d["_id"] || d["id"]}}
-        action_meta["index"]["_parent"] = d["_parent"] if d["_parent"]
-        post_data << (action_meta.to_json + "\n")
-        post_data << (d.to_json + "\n")
+        index_meta = { :_index => name, :_id => (d.delete(:id) || d.delete(:_id)) }
+        
+        d.keys.reduce(index_meta) do |memo, key|
+          index_meta[key] = d.delete(key) if key.to_s.start_with?('_')
+        end
+
+        post_data << ({action => index_meta}.to_json + "\n")
+        post_data << (d.to_json + "\n") unless action == :delete
+        post_data
       }
+      @server.bulk body, options
     end
 
     # Creates the index, with the supplied hash as the optinos body (usually mappings: and settings:))
     def create(options={})
-      request(:put) do |req|
-        req.body = options
-      end
+      request(:put, "/", {}, options)
     end
 
     # Deletes the index

data/lib/stretcher/index_type.rb

@@ -26,40 +26,29 @@ module Stretcher
         options = {}
       end
       
-      # If fields is passed in as an array, properly encode it
-      arg_fields = options[:fields]
-      if arg_fields.is_a?(Array)
-        # no #merge! we don't want to mutate args
-        options = options.merge(:fields => arg_fields.join(","))
-      end
-      
       res = request(:get, id, options)
       raw ? res : (res["_source"] || res["fields"])
     end
 
     # Retrieves multiple documents of the index type by ID
     # http://www.elasticsearch.org/guide/reference/api/multi-get/
-    def mget(ids)
-      request(:get, '_mget') do |req|
-        req.body = { :ids => ids }
-      end
+    def mget(ids, options={})
+      request(:get, '_mget', options, :ids => ids)
     end
 
     # Explains a query for a specific document
-    def explain(id, query)
-      request(:get, "#{id}/_explain") do |req|
-        req.body = query
-      end
+    def explain(id, query, options={})
+      request(:get, "#{id}/_explain", options, query)
     end
 
     # Index an item with a specific ID
-    def put(id, source)
-      request(:put, id, source)
+    def put(id, source, options={})
+      request(:put, id, options, source)
     end
 
     # Index an item with automatic ID generation
-    def post(source)
-      request(:post, nil, source)
+    def post(source, options={})
+      request(:post, nil, options, source)
     end
 
     # Uses the update api to modify a document with a script
@@ -69,12 +58,12 @@ module Stretcher
     # Takes an optional, third options hash, allowing you to specify
     # Additional query parameters such as +fields+ and +routing+
     def update(id, body, options={})
-      request(:post, Util.qurl("#{id}/_update", options), body)
+      request(:post, "#{id}/_update", options, body)
     end
 
     # Deletes the document with the given ID
-    def delete(id)
-      request :delete, id
+    def delete(id, options={})
+      request :delete, id, options
     rescue Stretcher::RequestError => e
       raise e if e.http_response.status != 404
       false
@@ -94,9 +83,7 @@ module Stretcher
 
     # Alter the mapping for this type
     def put_mapping(body)
-      request(:put, "_mapping") {|req|
-        req.body = body
-      }
+      request(:put, "_mapping", {}, body)
     end
 
     # Check if this index type is defined, if passed an id

data/lib/stretcher/server.rb

@@ -11,8 +11,8 @@ module Stretcher
 
         builder.request :json
 
-        builder.options[:read_timeout] = 4 || options[:read_timeout]
-        builder.options[:open_timeout] = 2 || options[:open_timeout]
+        builder.options[:read_timeout] = options[:read_timeout] || 4
+        builder.options[:open_timeout] = options[:open_timeout] || 2
 
         if faraday_configurator = options[:faraday_configurator]
           faraday_configurator.call(builder)
@@ -20,6 +20,11 @@ module Stretcher
           builder.adapter :excon
         end
       end
+      http.headers = {
+        :accept =>  'application/json',
+        :user_agent => "Stretcher Ruby Gem #{Stretcher::VERSION}",
+        "Content-Type" => "application/json"
+      }
 
       uri_components = URI.parse(uri)
       if uri_components.user || uri_components.password
@@ -90,10 +95,8 @@ module Stretcher
     
     # Perform a raw bulk operation. You probably want to use Stretcher::Index#bulk_index
     # which properly formats a bulk index request.
-    def bulk(data)
-      request(:post, path_uri("/_bulk")) do |req|
-        req.body = data
-      end
+    def bulk(data, options={})
+      request(:post, path_uri("/_bulk"), options, data)
     end
 
     # Retrieves stats for this server
@@ -125,12 +128,10 @@ module Stretcher
     def msearch(body=[])
       raise ArgumentError, "msearch takes an array!" unless body.is_a?(Array)
       fmt_body = body.map(&:to_json).join("\n") + "\n"
-      logger.info { "Stretcher msearch: curl -XGET #{uri} -d '#{fmt_body}'" }
-      res = request(:get, path_uri("/_msearch")) {|req|
-        req.body = fmt_body
-      }
+      
+      res = request(:get, path_uri("/_msearch"), {}, fmt_body)
 
-      errors = res.responses.select {|r| r[:error]}.map(&:error)
+      errors = res.responses.map(&:error).compact
       if !errors.empty?
         raise RequestError.new(res), "Could not msearch #{errors.inspect}"
       end
@@ -163,9 +164,7 @@ module Stretcher
     # as per: http://www.elasticsearch.org/guide/reference/api/admin-indices-aliases.html
     def aliases(body=nil)
       if body
-        request(:post, path_uri("/_aliases")) do |req|
-          req.body = body
-        end
+        request(:post, path_uri("/_aliases"), {}, body)
       else
         request(:get, path_uri("/_aliases"))
       end
@@ -182,24 +181,25 @@ module Stretcher
     end
 
     # Handy way to query the server, returning *only* the body
-    # Will raise an exception when the status is not in the 2xx range
-    def request(method, url=nil, query_opts=nil, *args, &block)
-      logger.info { "Stretcher: Issuing Request #{method.to_s.upcase}, #{Util.qurl(url,query_opts)}" }
-      
-      # Our default client is threadsafe, but some others might not be
-      check_response(@request_mtx.synchronize {
-        if block
-          http.send(method, url, query_opts, *args) do |req|
-            # Default content type to json, the block can change this of course
-            req.headers["Content-Type"] = 'application/json' unless req.headers
-            block.call(req)
-          end
-        else
-          http.send(method, url, query_opts, *args)
-        end
-      })
+    # Will raise an exception when the status is not in the 2xx range    
+    def request(method, path, params={}, body=nil, headers={}, &block)
+      req = http.build_request(method)
+      req.path = path
+      req.params.update(Util.clean_params(params)) if params
+      req.body = body if body
+      req.headers.update(headers) if headers
+      block.call(req) if block
+      logger.debug { Util.curl_format(req) }
+
+      @request_mtx.synchronize {
+        env = req.to_env(http)
+        check_response(http.app.call(env))
+      }
     end
-    
+
+    private
+
+
     # Internal use only
     # Check response codes from request
     def check_response(res)

data/lib/stretcher/util.rb

@@ -1,5 +1,17 @@
 module Stretcher
   module Util
+    
+    # cURL formats a Faraday req. Useful for logging
+    def self.curl_format(req)
+      body = "-d '#{req.body.is_a?(Hash) ? req.body.to_json : req.body}'" if req.body
+      headers = req.headers.map {|name, value| "'-H #{name}: #{value}'" }.sort.join(' ')
+      method = req.method.to_s.upcase
+      url = Util.qurl(req.path,req.params)
+      
+      ["curl -X#{method}", url, body, headers].compact.join(' ')  
+    end
+
+    # Formats a url + query opts
     def self.qurl(url, query_opts=nil)
       query_opts && !query_opts.empty? ? "#{url}?#{querify(query_opts)}" : url
     end
@@ -7,5 +19,15 @@ module Stretcher
     def self.querify(hash)
       hash.map {|k,v| "#{k}=#{v}"}.join('&')
     end
+
+    def self.clean_params params={}
+      return unless params
+      clean_params = {}
+      params.each do |key, value|
+        clean_params[key] = value.is_a?(Array) ? value.join(',') : value
+      end
+      clean_params
+    end
+
   end
 end

data/lib/stretcher/version.rb

@@ -1,3 +1,3 @@
 module Stretcher
-  VERSION = "1.13.0"
+  VERSION = "1.14.0"
 end

data/spec/lib/stretcher_index_spec.rb

@@ -11,7 +11,12 @@ describe Stretcher::Index do
     rescue Stretcher::RequestError::NotFound
     end
     server.refresh
-    i.create
+    i.create({
+      :settings => {
+        :number_of_shards => 1,
+        :number_of_replicas => 0
+      }
+    })
     # Why do both? Doesn't hurt, and it fixes some races
     server.refresh
     i.refresh
@@ -66,14 +71,57 @@ describe Stretcher::Index do
   end
 
   it "should retrieve settings properly" do
-    index.get_settings['foo']['settings']['index.number_of_replicas'].should_not be_nil
+    index.get_settings['foo']['settings']['index.number_of_shards'].should eq("1")
+    index.get_settings['foo']['settings']['index.number_of_replicas'].should eq("0")
   end
 
-  it "should bulk index documents properly" do
-    seed_corpus
-    corpus.each {|doc|
-      index.type(doc["_type"]).get(doc["_id"] || doc["id"]).text.should == doc[:text]
-    }
+  describe "bulk operations" do
+    it "should bulk index documents properly" do
+      seed_corpus
+      corpus.each {|doc|
+        fetched_doc = index.type(doc["_type"]).get(doc["_id"] || doc["id"], {}, true)
+        fetched_doc._source.text.should == doc[:text]
+        fetched_doc._source._id.should be_nil
+        fetched_doc._source._type.should be_nil
+      }
+    end
+
+    it "should bulk delete documents" do
+      seed_corpus
+      index.bulk_delete([
+                         {"_type" => 'tweet', "_id" => 'fooid'},
+                         {"_type" => 'tweet', "_id" => 'barid'},
+                        ])
+      index.refresh
+      res = index.search({}, {:query => {:match_all => {}}})
+      expect(res.results.map(&:_id)).to match_array ['bazid']
+    end
+  end
+
+  it 'allows _routing to be set on bulk index documents' do
+    server.index(:with_routing).delete if server.index(:with_routing).exists?
+    server.index(:with_routing).create({
+      :settings => {
+        :number_of_shards => 1,
+        :number_of_replicas => 0
+      },
+      :mappings => {
+        :_default_ =>  {
+          :_routing => { :required => true }
+        }
+      }
+    })
+
+    lambda {server.index(:with_routing).bulk_index(corpus)}.should raise_exception
+    routed_corpus = corpus.map do |doc|
+      routed_doc = doc.clone
+      routed_doc['_routing'] = 'abc'
+      routed_doc
+    end
+
+    server.index(:with_routing).bulk_index(routed_corpus)
+
+    server.index(:with_routing).delete
   end
 
   it "should delete by query" do
@@ -93,11 +141,37 @@ describe Stretcher::Index do
   end
 
   # TODO: Actually use two indexes
-  it "should msearch across the index returning all docs" do
-    seed_corpus
-    res = index.msearch([{:query => {:match_all => {}}}])
-    res.length.should == 1
-    res[0].class.should == Stretcher::SearchResults
+  describe "msearch" do
+    let(:res) {
+      seed_corpus
+      q2_text = corpus.first[:text]
+      queries = [
+                 {:query => {:match_all => {}}},
+                 {:query => {:match => {:text => q2_text}}}
+                ]
+      index.msearch(queries)
+    }
+    
+    it "should return an array of Stretcher::SearchResults" do
+      res.length.should == 2
+      res[0].class.should == Stretcher::SearchResults
+    end
+
+    it "should return the query results in order" do
+      # First query returns all docs, second only one
+      res[0].results.length.should == corpus.length
+      res[1].results.length.should == 1
+    end
+
+    it "should raise an error if any query is bad" do
+      queries = [
+                 {:query => {:match_all => {}}},
+                 {:query => {:invalid_query => {}}}
+                ]
+      expect {
+        index.msearch(queries)
+      }.to raise_error(Stretcher::RequestError)
+    end
   end
 
   it "execute the analysis API and return an expected result" do

data/spec/lib/stretcher_index_type_spec.rb

@@ -1,19 +1,10 @@
 require 'spec_helper'
 
 describe Stretcher::IndexType do
-  let(:server) { Stretcher::Server.new(ES_URL) }
-  let(:index) {
-    i = server.index(:foo)
-    i
-  }
-  let(:type) {
-    t = index.type(:bar)
-    t.delete_query(:match_all => {})
-    index.refresh
-    mapping = {"bar" => {"properties" => {"message" => {"type" => "string"}}}}
-    t.put_mapping mapping
-    t
-  }
+  let(:server) { Stretcher::Server.new(ES_URL, :logger => DEBUG_LOGGER) }
+  let(:index) { server.index(:foo) }
+  
+  let(:type) {  index.type(:bar) }
 
   it "should be existentially aware" do
     t = index.type(:existential)
@@ -24,9 +15,16 @@ describe Stretcher::IndexType do
     t.get_mapping.should == mapping
   end
 
+  before do
+    type.delete_query(:match_all => {})
+    index.refresh
+    type.put_mapping({"bar" => {"properties" => {"message" => {"type" => "string"}}}})
+  end
+
   describe "searching" do
     before do
       @doc = {:message => "hello"}
+
       type.put(123123, @doc)
       index.refresh
     end
@@ -67,6 +65,12 @@ describe Stretcher::IndexType do
       type.mget([988, 989]).docs.first._source.message.should == 'message one!'
       type.mget([988, 989]).docs.last._source.message.should == 'message two!'
     end
+
+    it 'allows options to be passed through' do
+      response = type.mget([988, 989], :fields => 'message')
+      response.docs.first.fields.message.should == 'message one!'
+      response.docs.last.fields.message.should == 'message two!'
+    end
   end
 
   describe "ops on individual docs" do
@@ -76,12 +80,12 @@ describe Stretcher::IndexType do
     end
     
     describe "put" do
-      it "should put correctly" do
-        @put_res.should_not be_nil
+      it "should put correctly, with options" do
+        type.put(987, @doc, :version_type => :external, :version => 42)._version.should == 42
       end
 
-      it "should post correctly" do
-        type.post(@doc).should_not be_nil
+      it "should post correctly, with options" do
+        type.post(@doc, :version_type => :external, :version => 42)._version.should == 42
       end
     end
 
@@ -127,18 +131,35 @@ describe Stretcher::IndexType do
       end
     end
     
-    it "should explain a query" do
-      type.exists?(987).should be_true
-      index.refresh
-      res = type.explain(987, {:query => {:match_all => {}}})
-      res.should have_key('explanation')
+    describe 'explain' do
+      it "should explain a query" do
+        type.exists?(987).should be_true
+        index.refresh
+        res = type.explain(987, {:query => {:match_all => {}}})
+        res.should have_key('explanation')
+      end
+
+      it 'should allow options to be passed through' do
+        index.refresh
+        type.explain(987, {:query => {:match_all => {}}}, {:fields => 'message'}).get.fields.message.should == 'hello!'
+      end
     end
     
-    it "should update individual docs correctly" do
+    it "should update individual docs correctly using ctx.source" do
       type.update(987, :script => "ctx._source.message = 'Updated!'")
       type.get(987).message.should == 'Updated!'
     end
 
+    it "should update individual docs correctly using doc" do
+      type.update(987, :doc => {:message => 'Updated!'})
+      type.get(987).message.should == 'Updated!'
+    end
+
+    it "should update individual docs correctly using doc and fields" do
+      response =  type.update(987, {:doc => {:message => 'Updated!'}}, :fields => 'message')
+      response.get.fields.message.should == 'Updated!'      
+    end
+
     it "should delete by query correctly" do
       type.delete_query("match_all" => {})
       index.refresh
@@ -150,5 +171,12 @@ describe Stretcher::IndexType do
       type.delete(987)
       type.exists?(987).should be_false
     end
+
+    it "should allow params to be passed to delete" do
+      version = type.get(987, {},  true)._version
+      lambda { type.delete(987, :version => version + 1) }.should raise_exception
+      type.delete(987, :version => version)
+      type.exists?(987).should be_false
+    end
   end
 end

data/spec/lib/stretcher_server_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Stretcher::Server do
-  let(:server) { Stretcher::Server.new(ES_URL) }
+  let(:server) { Stretcher::Server.new(ES_URL, :logger => DEBUG_LOGGER) }
 
   it "should initialize cleanly" do
     server.class.should == Stretcher::Server
@@ -38,8 +38,6 @@ describe Stretcher::Server do
   end
 
   it "should perform alias operations properly" do
-    # Tear down any leftovers from previous runs
-    server.aliases(:actions => [{:remove => {:alias => :foo_alias}}]) if server.index(:foo_alias).exists?
     server.index(:foo).delete if server.index(:foo).exists?
     server.index(:foo).create
     server.aliases(:actions => [{:add => {:index => :foo, :alias => :foo_alias}}])
@@ -75,4 +73,11 @@ describe Stretcher::Server do
     analyzed = server.analyze("Candles", :analyzer => :snowball)
     analyzed.tokens.first.token.should == 'candl'
   end
+
+  it 'logs requests correctly' do
+    server.logger.should_receive(:debug) do |&block|
+      block.call.should == %{curl -XGET http://localhost:9200/_analyze?analyzer=snowball -d 'Candles' '-H Accept: application/json' '-H Content-Type: application/json' '-H User-Agent: Stretcher Ruby Gem #{Stretcher::VERSION}'}
+    end
+    server.analyze("Candles", :analyzer => :snowball)
+  end
 end

data/spec/spec_helper.rb

@@ -4,6 +4,7 @@ Coveralls.wear!
 require 'rspec'
 require 'stretcher'
 
+File.open("test_logs", 'wb') {|f| f.write("")}
 DEBUG_LOGGER = Logger.new('test_logs')
 DEBUG_LOGGER.level = Logger::DEBUG
 ES_URL = 'http://localhost:9200'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: stretcher
 version: !ruby/object:Gem::Version
-  version: 1.13.0
+  version: 1.14.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-05 00:00:00.000000000 Z
+date: 2013-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday