waistband 0.2.5 → 0.3.0

data/.rspec

@@ -1,2 +1 @@
---format nested
---color
+--color

data/README.md

@@ -31,17 +31,19 @@ Configuration is generally pretty simple.  First, create a folder where you'll s
 ```yml
 # #{APP_DIR}/config/waistband/waistband.yml
 development:
+  timeout: 2
   servers:
     server1:
       host: http://localhost
       port: 9200
 ```
 
-You can name the servers whatever you want, and one of them is selected at random using `Array.sample` when initializing the configuration singleton.  Here's an example with two servers:
+You can name the servers whatever you want, and one of them is selected at random using `Array.sample`, excluding blacklisted servers, when conduction operations on the server.  Here's an example with two servers:
 
 ```yml
 # #{APP_DIR}/config/waistband/waistband.yml
 development:
+  timeout: 2
   servers:
     server1:
       host: http://173.247.192.214

data/lib/waistband/configuration.rb

@@ -1,6 +1,7 @@
 require 'yaml'
 require 'singleton'
 require 'active_support/core_ext/hash/indifferent_access'
+require 'digest/sha1'
 
 module Waistband
   class Configuration
@@ -26,26 +27,26 @@ module Waistband
       @indexes[name] ||= YAML.load_file("#{config_dir}/waistband_#{name}.yml")[@env].with_indifferent_access
     end
 
-    def hostname
-      "#{host}:#{port}"
-    end
-
     def method_missing(method_name, *args, &block)
       return current_server[method_name]  if current_server[method_name]
       return @yml_config[method_name]     if @yml_config[method_name]
       super
     end
 
+    def servers
+      @servers ||= @yml_config['servers'].map do |server_name, config|
+        config.merge({
+          '_id' => Digest::SHA1.hexdigest("#{config['host']}:#{config['port']}")
+        })
+      end
+    end
+
     private
 
       def current_server
         servers.sample
       end
 
-      def servers
-        @servers ||= @yml_config['servers'].map {|server_name, config| config}
-      end
-
     # /private
 
   end

data/lib/waistband/connection.rb

@@ -0,0 +1,129 @@
+require 'json'
+require 'rest-client'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Waistband
+  class Connection
+
+    class NoMoreServers < Exception; end
+
+    def initialize(options = {})
+      @blacklist      = []
+      @retry_on_fail  = options.fetch :retry_on_fail, true
+      @orderly        = options.fetch :orderly, false
+      pick_server
+    end
+
+    def create!(index)
+      execute! 'post', relative_url_for_index(index), index_json(index)
+    rescue RestClient::BadRequest => ex
+      nil
+    end
+
+    def destroy!(index)
+      execute! 'delete', relative_url_for_index(index)
+    rescue RestClient::ResourceNotFound => ex
+      nil
+    end
+
+    def update_settings!(index)
+      execute! 'put', "#{relative_url_for_index(index)}/_settings", settings_json(index)
+    end
+
+    def refresh(index)
+      execute! 'post', "#{relative_url_for_index(index)}/_refresh", {}
+    end
+
+    def read(index, key)
+      fetched = execute! 'get', relative_url_for_key(index, key)
+      JSON.parse(fetched)['_source'].with_indifferent_access
+    rescue RestClient::ResourceNotFound => ex
+      nil
+    end
+
+    def put(index, key, data)
+      execute! 'put', relative_url_for_key(index, key), data.to_json
+    end
+
+    def delete!(index, key)
+      execute! 'delete', relative_url_for_key(index, key)
+    end
+
+    def search_url_for_index(index)
+      "#{url}/#{relative_url_for_index(index)}/_search"
+    end
+
+    private
+
+      def execute!(method_name, relative_url, data = nil)
+        full_url = "#{url}/#{relative_url}"
+
+        Timeout::timeout ::Waistband.config.timeout do
+          RestClient.send method_name, full_url, data
+        end
+      rescue Timeout::Error, Errno::EHOSTUNREACH, Errno::ECONNREFUSED => e
+        # something's wrong, lets blacklist this sucker
+        blacklist! @server
+        retry if @retry_on_fail
+      end
+
+      def relative_url_for_key(index, key)
+        "#{relative_url_for_index(index)}/#{index.singularize}/#{key}"
+      end
+
+      def relative_url_for_index(index)
+        "#{index_name(index)}"
+      end
+
+      def url
+        "#{@server['host']}:#{@server['port']}"
+      end
+
+      def index_name(index)
+        config(index)['name']
+      end
+
+      def index_json(index)
+        config(index).except('name', 'stringify').to_json
+      end
+
+      def settings_json(index)
+        settings = config(index)['settings']['index'].except('number_of_shards')
+        {index: settings}.to_json
+      end
+
+      def config(index)
+        Waistband.config.index(index)
+      end
+
+      def blacklist!(server)
+        @blacklist << server['_id'] unless @blacklist.include? server['_id']
+        @blacklist
+
+        pick_server
+      end
+
+      def pick_server
+        @server = next_server
+
+        unless @server
+          raise ::Waistband::Connection::NoMoreServers.new "No available servers remain"
+        end
+
+        @server
+      end
+
+      def next_server
+        return available_servers.first if @orderly
+        available_servers.sample
+      end
+
+      def available_servers
+        ::Waistband.config.servers.reject {|server| @blacklist.include? server['_id']}
+      end
+
+    # /private
+
+  end
+end

data/lib/waistband/index.rb

@@ -1,42 +1,29 @@
-require 'json'
-require 'rest-client'
-require 'active_support/core_ext/string/inflections'
-require 'active_support/core_ext/hash/indifferent_access'
-require 'active_support/core_ext/hash/except'
-
 module Waistband
   class Index
 
-    MAX_RETRIES = 10
-
     def initialize(index)
       @index        = index
       @index_name   = config['name']
       @stringify    = config['stringify']
-      @retries      = 0
     end
 
     # create the index
     def create!
-      RestClient.post(url, index_json)
-    rescue RestClient::BadRequest => ex
-      nil
+      connection.create! @index
     end
 
     # destroy the index
     def destroy!
-      RestClient.delete(url)
-    rescue RestClient::ResourceNotFound => ex
-      nil
+      connection.destroy! @index
     end
 
     def update_settings!
-      RestClient.put("#{url}/_settings", settings_json)
+      connection.update_settings! @index
     end
 
     # refresh the index
     def refresh
-      RestClient.post("#{url}/_refresh", {})
+      connection.refresh @index
     end
 
     def store!(key, data)
@@ -44,59 +31,45 @@ module Waistband
       if @stringify
         original_data = data
 
-        if data.is_a?(Array)
-          data = Waistband::StringifiedArray.new(data)
-        elsif data.is_a?(Hash)
-          data = Waistband::StringifiedHash.new_from(data)
+        if data.is_a? Array
+          data = ::Waistband::StringifiedArray.new data
+        elsif data.is_a? Hash
+          data = ::Waistband::StringifiedHash.new_from data
         end
 
-        data = data.stringify_all if data.respond_to?(:stringify_all)
+        data = data.stringify_all if data.respond_to? :stringify_all
       end
 
-      result  = RestClient.put(url_for_key(key), data.to_json)
+      result  = connection.put @index, key, data
       data    = original_data if @stringify
 
       result
     end
 
     def delete!(key)
-      RestClient.delete(url_for_key(key))
+      connection.delete! @index, key
     end
 
     def read(key)
-      fetched = RestClient.get(url_for_key(key))
-      JSON.parse(fetched)['_source'].with_indifferent_access
-    rescue RestClient::ResourceNotFound => ex
-      nil
+      connection.read @index, key
     end
 
     def query(term, options = {})
-      Waistband::Query.new(@index_name, term, options)
+      ::Waistband::Query.new @index, term, options
     end
 
-    private
-
-      def url_for_key(key)
-        "#{url}/#{@index.singularize}/#{key}"
-      end
+    def search_url
+      connection.search_url_for_index @index
+    end
 
-      def settings_json
-        @settings_json ||= begin
-          settings = config['settings']['index'].except('number_of_shards')
-          {index: settings}.to_json
-        end
-      end
+    private
 
-      def index_json
-        @index_json ||= config.except('name', 'stringify').to_json
+      def connection
+        ::Waistband::Connection.new
       end
 
       def config
-        @config ||= Waistband.config.index(@index)
-      end
-
-      def url
-        "#{Waistband.config.hostname}/#{@index_name}"
+        Waistband.config.index @index
       end
 
     # /private

data/lib/waistband/query.rb

@@ -99,12 +99,16 @@ module Waistband
 
     private
 
-      def execute!
-        @executed ||= JSON.parse(RestClient.post(url, to_hash.to_json))
+      def url
+        index.search_url
       end
 
-      def url
-        "#{Waistband.config.hostname}/#{@index}/_search"
+      def index
+        Waistband::Index.new(@index)
+      end
+
+      def execute!
+        @executed ||= JSON.parse(RestClient.post(url, to_hash.to_json))
       end
 
       def to_hash

data/lib/waistband/version.rb

@@ -1,3 +1,3 @@
 module Waistband
-  VERSION = "0.2.5"
+  VERSION = "0.3.0"
 end

data/lib/waistband.rb

@@ -3,6 +3,7 @@ require "waistband/version"
 module Waistband
   
   autoload :Configuration,    "waistband/configuration"
+  autoload :Connection,       "waistband/connection"
   autoload :StringifiedArray, "waistband/stringified_array"
   autoload :StringifiedHash,  "waistband/stringified_hash"
   autoload :QueryResult,      "waistband/query_result"

data/spec/config/waistband/waistband.yml

@@ -1,10 +1,11 @@
 development: &DEV
+  timeout: 2
   servers:
     server1:
       host: http://localhost
       port: 9200
     server2:
-      host: http://localhost
+      host: http://127.0.0.1
       port: 9200
 
 test:

data/spec/lib/configuration_spec.rb

@@ -5,8 +5,9 @@ describe Waistband::Configuration do
   let(:config) { Waistband.config }
 
   it "loads config yml" do
-    config.host.should eql 'http://localhost'
+    config.host.should match /http\:\/\//
     config.port.should eql 9200
+    config.timeout.should eql 2
   end
 
   it "loads indexes config" do
@@ -21,4 +22,25 @@ describe Waistband::Configuration do
     config.index('events')['settings']['index']['number_of_shards'].should eql 4
   end
 
+  describe '#servers' do
+
+    it "returns array of all available servers' configs" do
+      config.servers.should be_an Array
+      config.servers.size.should eql 2
+
+      config.servers.each_with_index do |server, i|
+        server['host'].should match /http\:\/\//
+        server['port'].should eql 9200
+
+        server['_id'].should be_present
+        server['_id'].length.should eql 40
+      end
+    end
+
+    it "servers ids should be unique" do
+      config.servers[0]['_id'].should_not eql config.servers[1]['_id']
+    end
+
+  end
+
 end

data/spec/lib/connection_spec.rb

@@ -0,0 +1,225 @@
+require 'spec_helper'
+
+describe Waistband::Connection do
+
+  let(:connection) { Waistband::Connection.new }
+
+  def blacklist_server!
+    connection.send(:blacklist!, Waistband.config.servers.first)
+  end
+
+  it "constructs the settings json" do
+    connection.send(:settings_json, 'events').should eql '{"index":{"number_of_replicas":1}}'
+  end
+
+  it "constructs the index json" do
+    connection.send(:index_json, 'events').should eql '{"settings":{"index":{"number_of_shards":4,"number_of_replicas":1}},"mappings":{"event":{"_source":{"includes":["*"]}}}}'
+  end
+
+  describe '#execute!' do
+
+    it "wraps directly to rest client" do
+      connection = Waistband::Connection.new(orderly: true)
+
+      RestClient.should_receive(:get).with('http://localhost:9200/somekey', nil).once
+      connection.send(:execute!, 'get', 'somekey')
+    end
+
+    describe 'failures' do
+
+      [Timeout::Error, Errno::EHOSTUNREACH, Errno::ECONNREFUSED].each do |exception|
+        it "blacklists the server when #{exception}" do
+          connection = Waistband::Connection.new(retry_on_fail: false)
+
+          RestClient.should_receive(:get).with(kind_of(String), nil).and_raise exception
+          connection.send(:execute!, 'get', 'somekey')
+
+          connection.send(:available_servers).size.should eql 1
+        end
+      end
+
+      it "blacklists correctly when server is not responding" do
+        ::Waistband.config.stub(:servers).and_return(
+          [
+            {
+              host: "http://localhost",
+              port: 9123,
+              _id: "567890a5ce74182e5cd123e299993ab510c56123"
+            }.with_indifferent_access,
+            {
+              host: "http://localhost",
+              port: 9200,
+              _id: "282f32a5ce74182e5cd628e298b93ab510c5660c"
+            }.with_indifferent_access
+          ]
+        )
+
+        connection = Waistband::Connection.new(orderly: true)
+        expect { connection.refresh('events') }.to_not raise_error
+      end
+
+      it "keeps retrying till out of servers when retry_on_fail is true" do
+        RestClient.should_receive(:get).with('http://localhost:9200/somekey', nil).once.and_raise(Timeout::Error)
+        RestClient.should_receive(:get).with('http://127.0.0.1:9200/somekey', nil).once.and_raise(Timeout::Error)
+
+        expect {
+          connection.send(:execute!, 'get', 'somekey')
+        }.to raise_error(
+          Waistband::Connection::NoMoreServers,
+          "No available servers remain"
+        )
+      end
+
+    end
+
+  end
+
+  describe '#relative_url_for_key' do
+
+    it "returns the relative url for a key" do
+      url = connection.send(:relative_url_for_key, 'search', 'key123')
+      url.should match /^search_test\/search\/key123$/
+
+      url = connection.send(:relative_url_for_key, 'events', '9986')
+      url.should match /^events_test\/event\/9986$/
+    end
+
+  end
+
+  describe '#relative_url_for_index' do
+
+    it "returns the url for an index" do
+      url = connection.send(:relative_url_for_index, 'search')
+      url.should match /^search_test$/
+
+      url = connection.send(:relative_url_for_index, 'events')
+      url.should match /^events_test$/
+    end
+
+  end
+
+  describe '#url' do
+
+    it "returns url string for the selected server" do
+      url = connection.send(:url)
+      url.should match /^http\:\/\//
+      url.should match /\:9200$/
+    end
+
+  end
+
+  describe '#pick_server' do
+
+    it "randomly picks a server" do
+      server = connection.send :pick_server
+      server['host'].should match /http\:\/\//
+      server['port'].should eql 9200
+    end
+
+    it "never picks blacklisted servers" do
+      blacklist_server!
+
+      200.times do
+        server = connection.send :pick_server
+
+        server['host'].should eql 'http://127.0.0.1'
+      end
+    end
+
+    it "blows up when no more servers remain" do
+      blacklist_server!
+
+      expect {
+        connection.send(:blacklist!, Waistband.config.servers.last)
+      }.to raise_error(
+        Waistband::Connection::NoMoreServers,
+        "No available servers remain"
+      )
+    end
+
+  end
+
+  describe '#blacklist!' do
+
+    it "blacklists a server" do
+      connection.instance_variable_get('@blacklist').should be_empty
+
+      blacklist_server!
+
+      connection.instance_variable_get('@blacklist').size.should eql 1
+      connection.instance_variable_get('@blacklist').first.should eql Waistband.config.servers.first['_id']
+    end
+
+    it "doesn't keep duplicate servers" do
+      connection.instance_variable_get('@blacklist').should be_empty
+
+      blacklist_server!
+
+      connection.instance_variable_get('@blacklist').size.should eql 1
+
+      blacklist_server!
+
+      connection.instance_variable_get('@blacklist').size.should eql 1
+    end
+
+  end
+
+  describe '#available_servers' do
+
+    it "returns an array of servers" do
+      connection.send(:available_servers).should be_an Array
+      connection.send(:available_servers).size.should eql 2
+    end
+
+    it "doesn't include blacklisted servers" do
+      blacklist_server!
+
+      connection.send(:available_servers).should be_an Array
+      connection.send(:available_servers).size.should eql 1
+
+      ids = connection.send(:available_servers).map{|s| s['_id']}
+      ids.should      include Waistband.config.servers.last['_id']
+      ids.should_not  include Waistband.config.servers.first['_id']
+    end
+
+  end
+
+  describe "storing" do
+
+    let(:index)   { ::Waistband::Index.new('events') }
+    let(:index2)  { ::Waistband::Index.new('search') }
+    let(:attrs)   { {'ok' => {'yeah' => true}} }
+
+    before { IndexHelper.prepare! }
+
+    it "stores data" do
+      connection.put('events', '__test_write', {'ok' => 'yeah'})
+      index.read('__test_write').should eql({'ok' => 'yeah'})
+    end
+
+    it "data is indirectly accessible" do
+      connection.put('events', '__test_not_string', attrs)
+      index.read('__test_not_string')[:ok][:yeah].should eql true
+    end
+
+    it "deletes data" do
+      connection.put('events', '__test_write', attrs)
+      connection.delete!('events', '__test_write')
+      index.read('__test_write').should be_nil
+    end
+
+    it "returns nil on 404" do
+      index.read('__not_here').should be_nil
+    end
+
+    it "doesn't mix data between two indexes" do
+      connection.put('events', '__test_write',  {'data' => 'index_1'})
+      connection.put('search', '__test_write', {'data' => 'index_2'})
+
+      index.read('__test_write').should   eql({'data' => 'index_1'})
+      index2.read('__test_write').should  eql({'data' => 'index_2'})
+    end
+
+  end
+
+end

data/spec/lib/index_spec.rb

@@ -9,7 +9,6 @@ describe Waistband::Index do
   it "initializes values" do
     index.instance_variable_get('@index_name').should eql 'events_test'
     index.instance_variable_get('@stringify').should  eql true
-    index.instance_variable_get('@retries').should    eql 0
   end
 
   it "creates the index" do
@@ -32,14 +31,6 @@ describe Waistband::Index do
     response['ok'].should be_true
   end
 
-  it "constructs the settings json" do
-    index.send(:settings_json).should eql '{"index":{"number_of_replicas":1}}'
-  end
-
-  it "constructs the index json" do
-    index.send(:index_json).should eql '{"settings":{"index":{"number_of_shards":4,"number_of_replicas":1}},"mappings":{"event":{"_source":{"includes":["*"]}}}}'
-  end
-
   it "proxies to a query" do
     index.query('shopping').should be_a Waistband::Query
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waistband
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2013-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,6 +108,7 @@ files:
 - Rakefile
 - lib/waistband.rb
 - lib/waistband/configuration.rb
+- lib/waistband/connection.rb
 - lib/waistband/index.rb
 - lib/waistband/model.rb
 - lib/waistband/query.rb
@@ -120,6 +121,7 @@ files:
 - spec/config/waistband/waistband_events.yml
 - spec/config/waistband/waistband_search.yml
 - spec/lib/configuration_spec.rb
+- spec/lib/connection_spec.rb
 - spec/lib/index_spec.rb
 - spec/lib/model_spec.rb
 - spec/lib/query_result_spec.rb
@@ -159,6 +161,7 @@ test_files:
 - spec/config/waistband/waistband_events.yml
 - spec/config/waistband/waistband_search.yml
 - spec/lib/configuration_spec.rb
+- spec/lib/connection_spec.rb
 - spec/lib/index_spec.rb
 - spec/lib/model_spec.rb
 - spec/lib/query_result_spec.rb