yoga_pants 0.1.0

data/lib/yoga_pants/client.rb

@@ -0,0 +1,144 @@
+module YogaPants
+  class Client
+    # This class will handle:
+    #   * connecting to ES nodes
+    #   * failing over to nodes in a list
+    #   * ES-specific error handling
+
+    class RequestError < RuntimeError
+      attr_reader :http_error
+
+      def initialize(message, http_error = nil)
+        @http_error = nil
+        super(message)
+      end
+    end
+
+    attr_accessor :hosts, :options, :active_host
+
+    def initialize(hosts, options = {})
+      @hosts       = [hosts].flatten.freeze # Accept 1 or more hosts
+      @options     = options
+      @max_retries = options[:max_retries] || 10
+      @retries     = 0
+      reset_hosts
+    end
+
+    def reset_hosts
+      @active_hosts = hosts.dup
+      @active_host = @active_hosts.shift
+    end
+
+    def get(path, args = {})
+      with_error_handling do
+        connection.get(path, args)
+      end
+    end
+
+    def post(path, args = {})
+      with_error_handling do
+        connection.post(path, args)
+      end
+    end
+
+    def put(path, args = {})
+      with_error_handling do
+        connection.put(path, args)
+      end
+    end
+
+    def delete(path, args = {})
+      with_error_handling do
+        connection.delete(path, args)
+      end
+    end
+
+    BULK_OPERATIONS_WITH_DATA = [:index, :create].freeze
+    def bulk(path, operations, args = {})
+      path = path.sub(%r{/(?:_bulk)?$}, '/_bulk')
+
+      with_error_handling do
+        payload = StringIO.new
+
+        operations.each do |action, metadata, data|
+          payload << JSON.dump({action => metadata})
+          payload << "\n"
+          if BULK_OPERATIONS_WITH_DATA.include?(action.to_sym)
+            payload << JSON.dump(data)
+            payload << "\n"
+          end
+        end
+
+        payload.rewind
+        connection.post(path, :query_string => args, :body => payload.read)
+      end
+    end
+
+    def multi_search(path, operations, args={})
+      path = path.sub(%r{/?(?:_msearch)?$}, '/_msearch')
+
+      with_error_handling do
+        payload = StringIO.new
+
+        operations.each do |header, body|
+          payload << JSON.dump(header) << "\n"
+          payload << JSON.dump(body) << "\n"
+        end
+
+        payload.rewind
+        connection.get(path, :query_string => args, :body => payload.read)
+      end
+    end
+
+    def exists?(path, args = {})
+      with_error_handling do
+        begin
+          if path.count("/") >= 3 # More than
+            connection.get(path, args)
+          else
+            connection.head(path).status_code == 200
+          end
+        rescue Connection::HTTPError => e
+          if e.status_code == 404
+            false
+          else
+            raise e
+          end
+        end
+      end
+    end
+
+    def reset
+      connection.reset
+    end
+
+    private
+
+    def connection
+      @connection ||= Connection.new(active_host, options[:connection])
+    end
+
+    def pick_next_host
+      @active_host = @active_hosts.shift
+      @connection = nil
+      reset_hosts if active_host.nil?
+    end
+
+    def with_error_handling(&block)
+      block.call.tap do
+        @retries = 0
+      end
+    rescue Connection::HTTPError => e
+
+      if @retries <= @max_retries
+        @retries += 1
+        pick_next_host
+        retry
+      elsif e.body.is_a?(Hash) && error = e.body['error']
+        raise RequestError.new("ElasticSearch Error: #{error}", e).tap { |ex| ex.set_backtrace(e.backtrace) }
+      else
+        raise RequestError.new(e.message, e).tap { |ex| ex.set_backtrace(e.backtrace) }
+      end
+    end
+  end
+end

data/lib/yoga_pants/connection.rb

@@ -0,0 +1,150 @@
+module YogaPants
+  class Connection
+    # This class is/will be an abstraction layer for the underlying
+    # HTTP client.
+    # TODO: Use https://github.com/rubiii/httpi so we don't have to deal
+    # with interfacing with multiple HTTP libraries
+
+    attr_accessor :host, :options
+
+    class HTTPError < RuntimeError
+      attr_reader :response, :status_code
+      def initialize(message, response = nil)
+        @response    = response
+        if response
+          @status_code = response.status_code
+          super(message + "\nBody: #{response.body}")
+        else
+          super(message)
+        end
+      end
+
+      def body
+        return nil if response.nil?
+        @body ||= begin
+          JSON.load(response.body)
+        rescue MultiJson::DecodeError
+          response.body
+        end
+      end
+    end
+
+    def initialize(host, options = {})
+      @host = host.chomp('/')
+      @options = options || {}
+      @http = HTTPClient.new
+
+      default_timeout      = @options[:timeout] || 5
+      http.connect_timeout = @options[:connect_timeout] || default_timeout
+      http.send_timeout    = @options[:send_timeout] || default_timeout
+      http.receive_timeout = @options[:receive_timeout] || default_timeout
+    end
+
+    # Body can be a string or hash
+
+    def get(path, args = {})
+      with_error_handling do
+        parse_arguments_and_handle_response(args) do |query_string, body|
+          http.get(url_for(path), :query => query_string, :body => body)
+        end
+      end
+    end
+
+    def post(path, args = {})
+      with_error_handling do
+        parse_arguments_and_handle_response(args) do |query_string, body|
+          response = http.post(url_for(path), :query => query_string, :body => body)
+        end
+      end
+    end
+
+    def put(path, args = {})
+      with_error_handling do
+        parse_arguments_and_handle_response(args) do |query_string, body|
+          response = http.put(url_for(path), :query => query_string, :body => body)
+        end
+      end
+    end
+
+    def delete(path, args = {})
+      with_error_handling do
+        parse_arguments_and_handle_response(args) do |query_string, body|
+          response = http.delete(url_for(path), :query => query_string, :body => body)
+        end
+      end
+    end
+
+    def head(path, args = {})
+      with_error_handling do
+        query_string, _ = parse_arguments(args)
+        http.head(url_for(path), :query => query_string)
+      end
+    end
+
+    def reset
+      http.reset_all
+    end
+
+    private
+
+    def parse_arguments_and_handle_response(args, &block)
+      query_string, body = parse_arguments(args)
+      parse_and_handle_response(
+        block.call(query_string, body)
+      )
+    end
+
+    def parse_and_handle_response(response)
+      case response.status_code
+      when 200..299
+        JSON.load(response.body)
+      else
+        raise HTTPError.new("Error performing HTTP request: #{response.status_code} #{response.reason}", response)
+      end
+    end
+
+    def parse_arguments(args)
+      [args[:query_string], jsonify_body(args[:body])]
+    end
+
+    def with_error_handling(&block)
+      block.call
+    rescue HTTPError => e
+      raise e
+    rescue Errno::ECONNREFUSED
+      raise HTTPError.new("Connection refused to #{host}")
+    rescue HTTPClient::ConnectTimeoutError
+      raise HTTPError.new("Connection timed out to #{host}")
+    rescue HTTPClient::SendTimeoutError
+      raise HTTPError.new("Request send timed out to #{host}")
+    rescue HTTPClient::ReceiveTimeoutError
+      raise HTTPError.new("Receive timed out from #{host}")
+    rescue => e
+      raise HTTPError.new("Unhandled exception within YogaPants::Connection: #{e} - #{e.message}").tap { |ex| ex.set_backtrace(e.backtrace) }
+    end
+
+    def jsonify_body(string_or_hash)
+      return nil if string_or_hash.nil?
+      case string_or_hash
+      when Hash
+        JSON.dump(string_or_hash)
+      when String
+        string_or_hash
+      else
+        raise ArgumentError.new("Unrecognised body class #{string_or_hash.class}")
+      end
+    end
+
+    def http
+      @http
+    end
+
+    def url_for(path)
+      if path[0..0] == "/"
+        "#{host}#{path}"
+      else
+        "#{host}/#{path}"
+      end
+    end
+  end
+end

data/lib/yoga_pants/json.rb

@@ -0,0 +1,25 @@
+require 'multi_json'
+
+# FIXME: This whole thing is cancer. tire uses multi_json 1.0.3 which
+# uses decode/encode.
+
+module YogaPants
+  class JSON
+    def self.load(*args)
+      if MultiJson.respond_to?(:decode)
+        MultiJson.decode(*args)
+      else
+        MultiJson.load(*args)
+      end
+    end
+
+    def self.dump(*args)
+      if MultiJson.respond_to?(:encode)
+        MultiJson.encode(*args)
+      else
+        MultiJson.dump(*args)
+      end
+    end
+
+  end
+end

data/lib/yoga_pants/version.rb

@@ -0,0 +1,3 @@
+module YogaPants
+  VERSION = "0.1.0"
+end

data/lib/yoga_pants.rb

@@ -0,0 +1,10 @@
+require "httpclient"
+require "multi_json"
+require "yoga_pants/version"
+require "yoga_pants/json"
+require "yoga_pants/connection"
+require "yoga_pants/client"
+
+module YogaPants
+  # Your code goes here...
+end

data/spec/integration/basic_spec.rb

@@ -0,0 +1,205 @@
+require "spec_helper"
+
+module YogaPants
+  describe "basic integration tests" do
+    subject do
+      Client.new("http://localhost:9200/")
+    end
+
+    before do
+      VCR.use_cassette('before_block') do
+        if subject.exists?("/yoga_pants_test")
+          subject.delete("/yoga_pants_test")
+        end
+
+        if subject.exists?("/yoga_pants_test_1")
+          subject.delete("/yoga_pants_test_1")
+        end
+
+        subject.post("/yoga_pants_test")
+        subject.put("/yoga_pants_test/doc/_mapping", :body => {
+          :doc => {
+            :properties => {
+               :foo => {:type => 'string'},
+               :bar => {:type => 'integer'}
+            }
+          }
+        })
+        subject.post("/yoga_pants_test/_refresh")
+      end
+    end
+
+    it "indexes a valid document" do
+      VCR.use_cassette('indexing') do
+        subject.post("/yoga_pants_test/doc/1", :body => {
+          :foo => 'bar'
+        })
+        subject.get("/yoga_pants_test/doc/1").should include({
+          '_index' => 'yoga_pants_test',
+          '_type' => 'doc',
+          '_id' => '1',
+          'exists' => true,
+          '_source' => {
+            'foo' => 'bar'
+          }
+        })
+      end
+    end
+
+    describe "bulk operations" do
+      it 'does bulk operations just fine' do
+        VCR.use_cassette('bulk') do
+          subject.bulk("/", [
+            [:index, {:_index => 'yoga_pants_test', :_type => 'doc', :_id => 2}, {:foo => 'hello bulk'}],
+            [:index, {:_index => 'yoga_pants_test_1', :_type => 'doc2', :_id => 2}, {:foo => 'hello bulk 2'}]
+          ], :refresh => true)
+
+          subject.get("/yoga_pants_test/doc/2").should include({
+            '_index' => 'yoga_pants_test',
+            '_type' => 'doc',
+            '_id' => '2',
+            'exists' => true,
+            '_source' => {
+              'foo' => 'hello bulk'
+            }
+          })
+
+          subject.get("/yoga_pants_test_1/doc2/2").should include({
+            '_index' => 'yoga_pants_test_1',
+            '_type' => 'doc2',
+            '_id' => '2',
+            'exists' => true,
+            '_source' => {
+              'foo' => 'hello bulk 2'
+            }
+          })
+        end
+      end
+
+      it 'does not break on per-document index errors' do
+        VCR.use_cassette('bulk_document_index_error') do
+          ret = subject.bulk("/", [
+            [:index, {:_index => 'yoga_pants_test', :_type => 'doc', :_id => 1}, {:bar => 1}],
+            [:index, {:_index => 'yoga_pants_test', :_type => 'doc', :_id => 2}, {:bar => 'invalid'}],
+          ])
+          errors = ret['items'].select { |operation| operation['index']['error'] }
+          errors.length.should == 1
+          errors.first["index"]["_id"].should == '2'
+          subject.exists?("/yoga_pants_test/doc/1").should be_true
+          subject.exists?("/yoga_pants_test/doc/2").should be_false
+        end
+      end
+
+      it 'throws an exception when ES barfs' do
+        VCR.use_cassette('bulk_error') do
+          expect do
+            subject.bulk("/", [
+              [:index, {:_index => 'yoga_pants_test', :_type => 'doc', :_id => 1}, {:bar => 1}],
+              [:index, {:_index => '', :_type => 'doc', :_id => 2}, {:bar => 'invalid'}],
+            ])
+          end.to raise_error(Client::RequestError, "ElasticSearch Error: ElasticSearchException[String index out of range: 0]; nested: StringIndexOutOfBoundsException[String index out of range: 0]; ")
+
+          subject.exists?("/yoga_pants_test/doc/1").should be_false
+          subject.exists?("/yoga_pants_test/doc/2").should be_false
+        end
+      end
+    end
+
+    describe "multi search" do
+      it 'does bulk operations just fine' do
+        VCR.use_cassette('multi_search') do
+          result = subject.multi_search("/", [
+            [{'index' => 'yoga_pants_test', 'type' => 'doc'}, {'query' => {'match_all' => {}}}],
+            [{'index' => 'yoga_pants_test_1', 'type' => 'doc2'}, {'query' => {'match_all' => {}}}],
+          ])
+
+          result.should have_key('responses')
+          result['responses'].size.should == 2
+
+        end
+      end
+    end
+
+    it "raises an exception on missing documents" do
+      VCR.use_cassette('missing') do
+        expect { subject.get("/yoga_pants_test/doc/not_exist") }.to raise_error(Client::RequestError, "Error performing HTTP request: 404 Not Found\nBody: #{'{"_index":"yoga_pants_test","_type":"doc","_id":"not_exist","exists":false}'}")
+      end
+    end
+
+    it "raises an exception on an invalid request" do
+      VCR.use_cassette('invalid_request') do
+        expect { subject.get("/this_does_not_exist") }.to raise_error(Client::RequestError, "Error performing HTTP request: 400 Bad Request\nBody: No handler found for uri [/this_does_not_exist] and method [GET]")
+      end
+    end
+
+    it "raises an exception when ES returns a hash with an error object" do
+      VCR.use_cassette('error') do
+        subject.post("/yoga_pants_test/doc/1", :body => {
+          :foo => 'bar'
+        })
+        expect { subject.get("/yoga_pants_test/doc/1/_mlt?min_term_freq=invalid") }.to raise_error(Client::RequestError, "ElasticSearch Error: Failed to parse int parameter [min_term_freq] with value [invalid]")
+      end
+    end
+
+  end
+
+  describe "network-related failures" do
+    let(:host) { "http://localhost:9200" }
+    subject do
+      Client.new(host, :connection => {:connect_timeout => 0.01})
+    end
+
+    context "connection refused" do
+      let(:host) { "http://localhost:1" } # Unlikely to be anything running on this port
+
+      it "raises an RequestError" do
+        VCR.use_cassette('connection_refused') do
+          expect { subject.exists?("/foo") }.to raise_error(Client::RequestError, "Connection refused to http://localhost:1")
+        end
+      end
+    end
+
+    context "connection timed out" do
+      let(:host) { "http://192.168.34.58" } # Ewww but yeah.
+
+      it "raises an RequestError" do
+        VCR.use_cassette('timed_out') do
+          expect { subject.exists?("/foo") }.to raise_error(Client::RequestError, "Connection timed out to #{host}")
+        end
+      end
+    end
+  end
+
+  describe "failing over to other nodes" do
+    subject do
+      Client.new(hosts)
+    end
+
+    describe "connection refused on first node" do
+      let(:hosts) { ["http://localhost:1/", "http://localhost:9200/"] }
+      it "automatically fails over" do
+        VCR.use_cassette('failover_refused') do
+          subject.exists?("/foo").should == false
+        end
+      end
+    end
+
+    describe "connection timed out on first node" do
+      let(:hosts) { ["http://10.13.37.3:1/", "http://localhost:9200/"] }
+      it "automatically fails over" do
+        VCR.use_cassette('failover_timeout') do
+          subject.exists?("/foo").should == false
+        end
+      end
+    end
+
+    describe "no working hosts" do
+      let(:hosts) { ["http://localhost:1/", "http://localhost:2/"] }
+      it "throws an exception" do
+        VCR.use_cassette('failover_impossible') do
+          expect { subject.exists?("/foo") }.to raise_error(Client::RequestError, /Connection refused to/)
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require 'vcr'
+require 'yoga_pants'
+
+VCR.configure do |c|
+  c.cassette_library_dir = 'fixtures/vcr_cassettes'
+  c.hook_into :webmock
+  c.allow_http_connections_when_no_cassette = true if ENV['ALLOW_NO_CASSETTE']
+  c.ignore_localhost = true if ENV['INTEGRATION']
+end

data/yoga_pants.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/yoga_pants/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jack Chen (chendo)"]
+  gem.description   = %q{A super lightweight interface to ElasticSearch's HTTP REST API}
+  gem.summary       = <<-TEXT.strip
+    A super lightweight interface to ElasticSearch's HTTP REST API.
+  TEXT
+  gem.homepage      = "https://github.com/chendo/yoga_pants"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "yoga_pants"
+  gem.require_paths = ["lib"]
+  gem.version       = YogaPants::VERSION
+
+  gem.add_runtime_dependency 'httpclient', '2.2.5'
+  gem.add_runtime_dependency 'multi_json'
+
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'vcr'
+  gem.add_development_dependency 'webmock'
+  gem.add_development_dependency 'rake'
+end