etcd-rb 1.0.0.pre0 → 1.0.0.pre1

data/README.md

@@ -1,4 +1,4 @@
-# Ruby CQL3 driver
+# Ruby [etcd](https://github.com/coreos/etcd) driver
 
 [![Build Status](https://travis-ci.org/iconara/etcd-rb.png?branch=master)](https://travis-ci.org/iconara/etcd-rb)
 [![Coverage Status](https://coveralls.io/repos/iconara/etcd-rb/badge.png?branch=master)](https://coveralls.io/r/iconara/etcd-rb)
@@ -7,6 +7,8 @@
 
 A modern Ruby, compatible with 1.9.3 or later. Continously tested with MRI 1.9.3, 2.0.0 and JRuby 1.7.x.
 
+An etcd cluster. _Currently incompatible with the most recent versions of etcd because they return the wrong URI for the leader._
+
 # Installation
 
     gem install etcd-rb --prerelease
@@ -21,7 +23,25 @@ client.set('/foo', 'bar')
 client.get('/foo')
 ```
 
-See the full [API documentation](http://rubydoc.info/gems/etcd-rb/frames) for more.
+See the full [API documentation](http://rubydoc.info/gems/etcd-rb/frames) for more. All core features are supported, including test-and-set, TTL, watches -- as well as a few convenience features like continuous watching.
+
+# Features
+
+## Continuous watches: observers
+
+Most of the time when you use watches with etcd you want to immediately re-watch the key when you get a change notification. The `Client#observe` method handles this for you, including re-watching with the last seen index, so that you don't miss any updates.
+
+## Automatic leader detection
+
+You can point the client to any node in the etcd cluster, it will ask that node for the current leader and direct all subsequent requests directly to the leader to avoid unnecessary redirects. When the leader changes, detected by a redirect, the new leader will be registered and used instead of the previous.
+
+## Automacic failover & retry
+
+When connecting for the first time, and when the leader changes, the list of nodes in the cluster is cached. Should the node that the client is talking to become unreachable, the client will attempt to connect to the next known node, until it finds one that responds. The first node to respond will be asked for the current leader, which will then be used for subsequent request.
+
+This is handled completely transparently to you.
+
+Watches are a special case, since they use long polling, they will break when the leader goes down. Observers will attempt to reestablish their watches with the new leader.
 
 # Changelog & versioning
 

data/lib/etcd/client.rb

@@ -59,13 +59,35 @@ module Etcd
   class Client
     # Creates a new `etcd` client.
     #
+    # You should call {#connect} on the client to properly initialize it. The
+    # creation of the client and connection is divided into two parts to avoid
+    # doing network connections in the object initialization code. Many times
+    # you want to defer things with side-effect until the whole object graph
+    # has been created. {#connect} returns self so you can just chain it after
+    # the call to {.new}, e.g. `Client.new.connect`.
+    #
+    # You can specify a seed node to connect to using the `:uri` option (which
+    # defaults to 127.0.0.1:4001), but once connected the client will prefer to
+    # talk to the master in order to avoid unnecessary HTTP requests, and to
+    # make sure that get operations find the most recent value.
+    #
     # @param [Hash] options
-    # @option options [String] :host ('127.0.0.1') The etcd host to connect to
-    # @option options [String] :port (4001) The port to connect to
+    # @option options [String] :uri ('http://127.0.0.1:4001') The etcd host and
+    #   port to connect to
     def initialize(options={})
-      @host = options[:host] || '127.0.0.1'
-      @port = options[:port] || 4001
+      @seed_uri = options[:uri] || 'http://127.0.0.1:4001'
+      @protocol_version = 'v1'
       @http_client = HTTPClient.new(agent_name: "etcd-rb/#{VERSION}")
+      @http_client.redirect_uri_callback = method(:handle_redirected)
+    end
+
+    def connect
+      change_uris(@seed_uri)
+      cache_machines
+      change_uris(@machines_cache.first)
+      self
+    rescue AllNodesDownError => e
+      raise ConnectionError, e.message, e.backtrace
     end
 
     # Sets the value of a key.
@@ -84,7 +106,7 @@ module Etcd
       if ttl = options[:ttl]
         body[:ttl] = ttl
       end
-      response = @http_client.post(uri(key), body)
+      response = request(:post, uri(key), body: body)
       data = MultiJson.load(response.body)
       data[S_PREV_VALUE]
     end
@@ -110,7 +132,7 @@ module Etcd
       if ttl = options[:ttl]
         body[:ttl] = ttl
       end
-      response = @http_client.post(uri(key), body)
+      response = request(:post, uri(key), body: body)
       response.status == 200
     end
 
@@ -123,7 +145,7 @@ module Etcd
     # @return [String, Hash] the value for the key, or a hash of keys and values
     #   when the key is a prefix.
     def get(key)
-      response = @http_client.get(uri(key))
+      response = request(:get, uri(key))
       if response.status == 200
         data = MultiJson.load(response.body)
         if data.is_a?(Array)
@@ -154,7 +176,7 @@ module Etcd
     # @return [Hash] a with info about the key, the exact contents depend on
     #   what kind of key it is.
     def info(key)
-      response = @http_client.get(uri(key))
+      response = request(:get, uri(key))
       if response.status == 200
         data = MultiJson.load(response.body)
         if data.is_a?(Array)
@@ -180,7 +202,7 @@ module Etcd
     # @param key [String] the key to remove
     # @return [String] the previous value, if any
     def delete(key)
-      response = @http_client.delete(uri(key))
+      response = request(:delete, uri(key))
       if response.status == 200
         data = MultiJson.load(response.body)
         data[S_PREV_VALUE]
@@ -228,7 +250,7 @@ module Etcd
       if index = options[:index]
         parameters[:index] = index
       end
-      response = @http_client.get(uri(prefix, S_WATCH), parameters)
+      response = request(:get, uri(prefix, S_WATCH), query: parameters)
       data = MultiJson.load(response.body)
       info = extract_info(data)
       yield info[:value], info[:key], info
@@ -268,6 +290,16 @@ module Etcd
       Observer.new(self, prefix, handler).tap(&:run)
     end
 
+    # Returns a list of URIs for the machines in the `etcd` cluster.
+    #
+    # The first URI is for the leader.
+    #
+    # @return [Array<String>] the URIs of the machines in the cluster
+    def machines
+      response = request(:get, @machines_uri)
+      response.body.split(MACHINES_SEPARATOR_RE)
+    end
+
     private
 
     S_KEY = 'key'.freeze
@@ -281,12 +313,23 @@ module Etcd
     S_PREV_VALUE = 'prevValue'.freeze
     S_ACTION = 'action'.freeze
     S_WATCH = 'watch'.freeze
+    S_LOCATION = 'location'.freeze
 
     S_SLASH = '/'.freeze
+    MACHINES_SEPARATOR_RE = /,\s*/
 
     def uri(key, action=S_KEYS)
       key = "/#{key}" unless key.start_with?(S_SLASH)
-      "http://#{@host}:#{@port}/v1/#{action}#{key}"
+      "#{@base_uri}/#{action}#{key}"
+    end
+
+    def request(method, uri, args={})
+      @http_client.request(method, uri, args.merge(follow_redirect: true))
+    rescue HTTPClient::TimeoutError => e
+      old_base_uri = @base_uri
+      handle_leader_down
+      uri.sub!(old_base_uri, @base_uri)
+      retry
     end
 
     def extract_info(data)
@@ -308,6 +351,32 @@ module Etcd
       info
     end
 
+    def handle_redirected(uri, response)
+      location = URI.parse(response.header[S_LOCATION][0])
+      change_uris("#{location.scheme}://#{location.host}:#{location.port}")
+      cache_machines
+      @http_client.default_redirect_uri_callback(uri, response)
+    end
+
+    def handle_leader_down
+      if @machines_cache && @machines_cache.any?
+        @machines_cache.reject! { |m| @base_uri.include?(m) }
+        change_uris(@machines_cache.shift)
+      else
+        raise AllNodesDownError, 'All known nodes are down'
+      end
+    end
+
+    def cache_machines
+      @machines_cache = machines
+    end
+
+    def change_uris(leader_uri, options={})
+      @base_uri = "#{leader_uri}/#{@protocol_version}"
+      @leader_uri = "#{@base_uri}/leader"
+      @machines_uri = "#{@base_uri}/machines"
+    end
+
     # @private
     class Observer
       def initialize(client, prefix, handler)

data/lib/etcd/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Etcd
-  VERSION = '1.0.0.pre0'.freeze
+  VERSION = '1.0.0.pre1'.freeze
 end

data/lib/etcd.rb

@@ -2,6 +2,8 @@
 
 module Etcd
   EtcdError = Class.new(StandardError)
+  ConnectionError = Class.new(EtcdError)
+  AllNodesDownError = Class.new(EtcdError)
 end
 
 require 'etcd/version'

data/spec/etcd/client_spec.rb

@@ -6,7 +6,7 @@ require 'spec_helper'
 module Etcd
   describe Client do
     let :client do
-      described_class.new(host: host, port: port)
+      described_class.new(uri: "http://#{host}:#{port}").connect
     end
 
     let :host do
@@ -14,13 +14,50 @@ module Etcd
     end
 
     let :port do
-      rand(2**16)
+      50_000
     end
 
     let :base_uri do
       "http://#{host}:#{port}/v1"
     end
 
+    let :machines_uri do
+      "http://#{host}:#{port}/v1/machines"
+    end
+
+    let :leader_uri do
+      "http://#{host}:#{port}/v1/leader"
+    end
+
+    let :machines do
+      %W[http://#{host}:#{port} http://#{host}:#{port + 2} http://#{host}:#{port + 1}]
+    end
+
+    before do
+      stub_request(:get, leader_uri).to_return(body: "http://#{host}:#{port}")
+      stub_request(:get, machines_uri).to_return(body: machines.join(', '))
+    end
+
+    describe '#connect' do
+      it 'gets the list of machines' do
+        client
+        WebMock.should have_requested(:get, machines_uri)
+      end
+
+      it 'raises an error when the seed node cannot be contacted' do
+        stub_request(:get, machines_uri).to_timeout
+        expect { client }.to raise_error(ConnectionError)
+      end
+
+      it 'discards the seed node and instead talks to the leader (the first in the list of machines)' do
+        machines.push(machines.shift)
+        stub_request(:get, machines_uri).to_return(body: machines.join(','))
+        stub_request(:get, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({'value' => 'wrong value'}))
+        stub_request(:get, "#{machines[0]}/v1/keys/foo").to_return(body: MultiJson.dump({'value' => 'right value'}))
+        client.get('/foo').should == 'right value'
+      end
+    end
+
     describe '#get' do
       before do
         stub_request(:get, "#{base_uri}/keys/foo").to_return(body: MultiJson.dump({'value' => 'bar'}))
@@ -55,6 +92,66 @@ module Etcd
           client.get('/foo').should eql({'/foo/bar' => 'bar', '/foo/baz' => 'baz'})
         end
       end
+
+      context 'when not talking to the master' do
+        before do
+          stub_request(:get, "#{base_uri}/keys/foo").to_return(status: 307, headers: {'Location' => 'http://example.com:7654/v1/keys/foo'})
+          stub_request(:get, 'http://example.com:7654/v1/machines').to_return(body: %w[http://example.com:7654 http://example.com:7655].join(', '))
+          stub_request(:get, 'http://example.com:7654/v1/keys/foo').to_return(body: MultiJson.dump({'value' => 'bar'}))
+        end
+
+        it 'follows redirects' do
+          client.get('/foo').should == 'bar'
+        end
+
+        it 'sets the new host as the new leader when it is redirected' do
+          client.get('/foo')
+          client.get('/foo').should == 'bar'
+          WebMock.should have_requested(:get, "#{base_uri}/keys/foo").once
+        end
+
+        it 'asks for a new list of cluster machines' do
+          client.get('/foo')
+          WebMock.should have_requested(:get, 'http://example.com:7654/v1/machines').once
+        end
+      end
+
+      context 'when nodes go down' do
+        let :uris do
+          machines.map { |m| "#{m}/v1/keys/thing" }
+        end
+
+        it 'tries with another node when the leader appears to be down' do
+          stub_request(:get, uris[0]).to_timeout
+          stub_request(:get, uris[1]).to_return(body: MultiJson.dump({'value' => 'bar'}))
+          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'baz'}))
+          client.get('/thing').should == 'bar'
+        end
+
+        it 'tries other nodes until it finds one that responds' do
+          stub_request(:get, uris[0]).to_timeout
+          stub_request(:get, uris[1]).to_timeout
+          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'bar'}))
+          client.get('/thing').should == 'bar'
+        end
+
+        it 'follows redirects when trying a node after a leader failure, and saves the new leader' do
+          stub_request(:get, uris[0]).to_timeout
+          stub_request(:get, uris[1]).to_return(status: 307, headers: {'Location' => uris[2]})
+          stub_request(:get, uris[2]).to_return(body: MultiJson.dump({'value' => 'bar'}))
+          stub_request(:get, uris[2].sub('/keys/thing', '/machines')).to_return(body: [uris[1], uris[2]].join(', '))
+          stub_request(:get, uris[2].sub('/thing', '/bar')).to_return(body: MultiJson.dump({'value' => 'baz'}))
+          client.get('/thing').should == 'bar'
+          client.get('/bar').should == 'baz'
+        end
+
+        it 'raises an error when all leader candidates have been exhausted' do
+          stub_request(:get, uris[0]).to_timeout
+          stub_request(:get, uris[1]).to_timeout
+          stub_request(:get, uris[2]).to_timeout
+          expect { client.get('/thing') }.to raise_error(AllNodesDownError)
+        end
+      end
     end
 
     describe '#set' do
@@ -341,5 +438,14 @@ module Etcd
         new_keys.should == [true, nil, nil]
       end
     end
+
+    describe '#machines' do
+      it 'returns a list of host and ports of the machines in the etcd cluster' do
+        body = 'http://host01:4001, http://host02:4001, http://host03:4001'
+        stub_request(:get, machines_uri).to_return(body: body)
+        stub_request(:get, machines_uri.sub(base_uri, 'http://host01:4001/v1')).to_return(body: body)
+        client.machines.should == %w[http://host01:4001 http://host02:4001 http://host03:4001]
+      end
+    end
   end
 end

data/spec/integration/etcd_spec.rb

@@ -0,0 +1,58 @@
+# encoding: utf-8
+
+require 'spec_helper'
+require 'open-uri'
+
+
+describe 'A etcd client' do
+  let :client do
+    Etcd::Client.new(uri: ENV['ETCD_URI']).connect
+  end
+
+  let :prefix do
+    "/etcd-rb/#{rand(234234)}"
+  end
+
+  let :key do
+    "#{prefix}/hello"
+  end
+
+  before do
+    WebMock.disable!
+  end
+
+  before do
+    begin
+      open("#{ENV['ETCD_URI']}/v1/leader").read
+    rescue Errno::ECONNREFUSED, Errno::ENOENT
+      pending('etcd not running, start it with `./spec/resources/etcd-cluster start`')
+    end
+  end
+
+  before do
+    client.delete(key)
+  end
+
+  it 'sets and gets the value for a key' do
+    client.set(key, 'foo')
+    client.get(key).should == 'foo'
+  end
+
+  it 'sets a key with a TTL' do
+    client.set(key, 'foo', ttl: 5)
+    client.info(key)[:ttl].should be_within(1).of(5)
+    client.info(key)[:expiration].should be_within(1).of(Time.now + 5)
+  end
+
+  it 'watches for changes to a key' do
+    Thread.start { sleep(0.1); client.set(key, 'baz') }
+    new_value = client.watch(key) { |value| value }
+    new_value.should == 'baz'
+  end
+
+  it 'conditionally sets the value for a key' do
+    client.set(key, 'bar')
+    client.update(key, 'qux', 'baz').should be_false
+    client.update(key, 'qux', 'bar').should be_true
+  end
+end

data/spec/spec_helper.rb

@@ -1,5 +1,24 @@
 # encoding: utf-8
 
 require 'bundler/setup'
+
+unless ENV['COVERAGE'] == 'no'
+  require 'coveralls'
+  require 'simplecov'
+
+  if ENV.include?('TRAVIS')
+    Coveralls.wear!
+    SimpleCov.formatter = Coveralls::SimpleCov::Formatter
+  end
+
+  SimpleCov.start do
+    add_group 'Source', 'lib'
+    add_group 'Unit tests', 'spec/etcd'
+    add_group 'Integration tests', 'spec/integration'
+  end
+end
+
+ENV['ETCD_URI'] ||= 'http://127.0.0.1:4001'
+
+require 'webmock/rspec'
 require 'etcd'
-require 'webmock/rspec'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: etcd-rb
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre0
+  version: 1.0.0.pre1
   prerelease: 6
 platform: ruby
 authors:
@@ -9,8 +9,40 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-04 00:00:00.000000000 Z
-dependencies: []
+date: 2013-08-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.7.0
 description: ''
 email:
 - theo@iconara.net
@@ -23,6 +55,7 @@ files:
 - lib/etcd.rb
 - README.md
 - spec/etcd/client_spec.rb
+- spec/integration/etcd_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/iconara/etcd-rb
 licenses:
@@ -45,11 +78,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: ''
 test_files:
 - spec/etcd/client_spec.rb
+- spec/integration/etcd_spec.rb
 - spec/spec_helper.rb
 has_rdoc: