etcdv3 0.9.0 → 0.11.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2c90b3d66d75efdf3c6c608cc1ccaf9673731070
-  data.tar.gz: 7698e02e69968d2e6348dd01a7e6813d27d5d50e
+SHA256:
+  metadata.gz: 18d76403d57c3e56b75e603931345900a4583b2e94d773b33ca04aa1547e304d
+  data.tar.gz: f1d1f9336f62f18b32fb9d3f055e0043bddb6076022b922b097c0104a7467c8b
 SHA512:
-  metadata.gz: 634ec36c4eccf0b93dd69b51a25e6515c06a26a1677264fb02d035c793cf905feb739ad9bde6e40ae30617416aebe7b85706bb4a1060cbb3d26d8b68e42a702f
-  data.tar.gz: 7043f69c71837cefc8168ff3d5c0225afc9109c4a1c5e9aab1bec7b5fcf56999c9866abac23c3a7b13e9de5422b1983ffae57448f85dafd65f5d369837d7d286
+  metadata.gz: 9633a47e982b7777ccda7e5472a663a661e4cf6368689cfbbd8d06a6ca862be47e7c3a75d075919b645611879513d4c88c7f59686b2a0eac15cc160e3ea7520f
+  data.tar.gz: 5a330de810e02ebdec73c279e33b7c2ece0c2aa7c4c877da89a2b285976ce68578ca20b3c51206d9a14c9fccbf18ef6310254010053eaefa59b06b898e1c195a

data/.gitignore

@@ -1,9 +1,10 @@
-Gemfile.lock
-Guardfile
-*.sh
 *.gem
-/tmp/
+*.sh
+*.swp
+/.bundle
 /.idea
 /coverage/*
 /doc/
-*.swp
+/tmp/
+Gemfile.lock
+Guardfile

data/.travis.yml

@@ -1,15 +1,14 @@
 language: ruby
 rvm:
+  - 3.0.2
   - 2.5.3
   - 2.4.5
   - 2.3.8
 
 env:
-  - ETCD_VERSION=v3.1.20
-  - ETCD_VERSION=v3.2.25
-# v3.3.10 is not working for whatever reason (at least in travis, spec passes
-# locally for me)
-#  - ETCD_VERSION=v3.3.10
+  - ETCD_VERSION_TO_TEST=v3.1.20
+  - ETCD_VERSION_TO_TEST=v3.2.25
+  - ETCD_VERSION_TO_TEST=v3.3.10
 
 install:
     - bundle install

data/README.md

@@ -26,6 +26,9 @@ conn = Etcdv3.new(endpoints: 'https://hostname:port')
 # Secure connection with Auth
 conn = Etcdv3.new(endpoints: 'https://hostname:port', user: 'root', password: 'mysecretpassword')
 
+# Scope CRUD operations to a specific keyspace. 
+conn = Etcdv3.new(endpoints: 'https://hostname:port', namespace: "/target_keyspace/")
+
 # Secure connection specifying custom certificates
 # Coming soon...
 
@@ -34,6 +37,43 @@ conn = Etcdv3.new(endpoints: 'https://hostname:port', user: 'root', password: 'm
 
 In the event of a failure, the client will work to restore connectivity by cycling through the specified endpoints until a connection can be established.  With that being said, it is encouraged to specify multiple endpoints when available.
 
+However, sometimes this is not what you want. If you need more control over
+failures, you can suppress this mechanism by using
+
+```ruby
+conn = Etcdv3.new(endpoints: 'https://hostname:port', allow_reconnect: false)
+```
+
+This will still rotate the endpoints, but it will raise an exception so you can
+handle the failure yourself. On next call to the new endpoint (since they were
+rotated) is tried. One thing you need to keep in mind if auth is enabled, you 
+need to take care of `GRPC::Unauthenticated` exception and manually re-authenticate 
+when token expires. To reiterate, you are responsible for handling the errors, so 
+some understanding of how this gem and etcd works is recommended.
+
+## Namespace support
+
+Namespacing is a convenience feature used to scope CRUD based operations to a specific keyspace.
+
+```ruby
+# Establish connection
+conn = Etcdv3.new(endpoints: 'https://hostname:port', namespace: '/service-a/')
+
+# Write key to /service-a/test_key
+conn.put("test_key", "value"). 
+
+# Get the key we just wrote.
+conn.get("test_key")
+
+# Retrieve everything under the namespace.
+conn.get("", "\0") 
+
+# Delete everything under the namespace.
+conn.del("", "\0")
+```
+
+_Note: Namespaces are stripped from responses._
+
 
 ## Adding, Fetching and Deleting Keys
 ```ruby
@@ -149,12 +189,15 @@ end
 
 ## Watch
 ```ruby
-# Watch for changes on a specified key and return
-events = conn.watch('foo')
+# Watch for changes on a specified key for at most 10 seconds and return
+events = conn.watch('foo', timeout: 10)
 
 # Watch for changes on a specified key range and return
 events = conn.watch('foo', range_end: 'fop')
 
+# Watch for changes since a given revision
+events = conn.watch('foo', start_revision: 42)
+
 # Watches for changes continuously until killed.
 event_count = 0
 conn.watch('boom') do |events|

data/Rakefile

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
-ETCD_VERSION = ENV["ETCD_VERSION"] || "v3.2.0"
-ETCD_URL = "https://github.com/coreos/etcd/releases/download/#{ETCD_VERSION}/etcd-#{ETCD_VERSION}-linux-amd64.tar.gz"
+ETCD_VERSION = ENV["ETCD_VERSION_TO_TEST"] || "v3.2.0"
+ETCD_URL = "https://github.com/coreos/etcd/releases/download/" \
+           "#{ETCD_VERSION}/etcd-#{ETCD_VERSION}-linux-amd64.tar.gz"
 
 require "tmpdir"
 

data/lib/etcdv3/connection.rb

@@ -1,6 +1,12 @@
 class Etcdv3
   class Connection
 
+    NAMESPACE_HANDLERS = {
+      kv: Etcdv3::Namespace::KV,
+      watch: Etcdv3::Namespace::Watch,
+      lock: Etcdv3::Namespace::Lock,
+    }
+
     HANDLERS = {
       auth: Etcdv3::Auth,
       kv: Etcdv3::KV,
@@ -10,18 +16,25 @@ class Etcdv3
       lock: Etcdv3::Lock,
     }
 
-    attr_reader :endpoint, :hostname, :handlers, :credentials
+    attr_reader :endpoint, :hostname, :handlers, :credentials, :namespace
 
-    def initialize(url, timeout, metadata={})
+    def initialize(url, timeout, namespace, metadata={})
       @endpoint = URI(url)
       @hostname = "#{@endpoint.hostname}:#{@endpoint.port}"
+      @namespace = namespace
       @credentials = resolve_credentials
       @timeout = timeout
       @handlers = handler_map(metadata)
     end
 
     def call(stub, method, method_args=[])
-      @handlers.fetch(stub).send(method, *method_args)
+      *method_args, method_kwargs = method_args if method_args.last.class == Hash
+
+      if method_kwargs.nil?
+        @handlers.fetch(stub).send(method, *method_args)
+      else
+        @handlers.fetch(stub).send(method, *method_args, **method_kwargs)
+      end
     end
 
     def refresh_metadata(metadata)
@@ -31,11 +44,19 @@ class Etcdv3
     private
 
     def handler_map(metadata={})
-      Hash[
+      handlers = Hash[
         HANDLERS.map do |key, klass|
-          [key, klass.new("#{@hostname}", @credentials, @timeout, metadata)]
+          [key, klass.new(@hostname, @credentials, @timeout, metadata)]
         end
       ]
+      # Override any handlers that are namespace compatable.
+      if @namespace
+        NAMESPACE_HANDLERS.each do |key, klass|
+          handlers[key] = klass.new(@hostname, @credentials, @timeout, @namespace, metadata)
+        end
+      end
+      
+      handlers
     end
 
     def resolve_credentials

data/lib/etcdv3/connection_wrapper.rb

@@ -3,31 +3,46 @@ class Etcdv3
 
     attr_accessor :connection, :endpoints, :user, :password, :token, :timeout
 
-    def initialize(timeout, *endpoints)
+    def initialize(timeout, *endpoints, namespace, allow_reconnect)
       @user, @password, @token = nil, nil, nil
       @timeout = timeout
-      @endpoints = endpoints.map{|endpoint| Etcdv3::Connection.new(endpoint, @timeout) }
+      @namespace = namespace
+      @endpoints = endpoints.map{|endpoint| Etcdv3::Connection.new(endpoint, @timeout, @namespace) }
+      @allow_reconnect = allow_reconnect
       @connection = @endpoints.first
     end
 
+    private def retry_or_raise(*args)
+      if @allow_reconnect
+        *args, kwargs = args if args.last.class == Hash
+
+        if kwargs.nil?
+          handle(*args)
+        else
+          handle(*args, **kwargs)
+        end
+      else
+        raise
+      end
+    end
+
     def handle(stub, method, method_args=[], retries: 1)
       @connection.call(stub, method, method_args)
 
-    rescue GRPC::Unavailable, GRPC::Core::CallError => exception
+    rescue GRPC::Unavailable, GRPC::Core::CallError
       $stderr.puts("Failed to connect to endpoint '#{@connection.hostname}'")
       if @endpoints.size > 1
         rotate_connection_endpoint
-        $stderr.puts("Failover event triggered. Failing over to '#{@connection.hostname}'")
-        return handle(stub, method, method_args)
+        return retry_or_raise(stub, method, method_args)
       else
-        return handle(stub, method, method_args)
+        return retry_or_raise(stub, method, method_args)
       end
     rescue GRPC::Unauthenticated => exception
       # Regenerate token in the event it expires.
       if exception.details == 'etcdserver: invalid auth token'
         if retries > 0
           authenticate(@user, @password)
-          return handle(stub, method, method_args, retries: retries - 1)
+          return retry_or_raise(stub, method, method_args, retries: retries - 1)
         end
       end
       raise exception

data/lib/etcdv3/namespace/kv/requests.rb

@@ -0,0 +1,54 @@
+class Etcdv3::Namespace::KV
+  module Requests
+
+    SORT_TARGET = {
+      key: 0,
+      version: 1,
+      create: 2,
+      mod: 3,
+      value: 4
+    }
+
+    SORT_ORDER = {
+      none: 0,
+      ascend: 1,
+      descend: 2
+    }
+
+    def get_request(key, opts)
+      key = prepend_prefix(@namespace, key)
+      # In order to enforce the scope of the specified namespace, we are going to
+      # intercept the zero-byte reference and re-target everything under the given namespace.
+      if opts[:range_end] =~ /\x00/
+        opts[:range_end] = (@namespace[0..-2] + (@namespace[-1].ord + 1).chr)
+      else 
+        opts[:range_end] = prepend_prefix(@namespace, opts[:range_end]) if opts[:range_end]
+      end
+      opts[:sort_order] = SORT_ORDER[opts[:sort_order]] \
+        if opts[:sort_order]
+      opts[:sort_target] = SORT_TARGET[opts[:sort_target]] \
+        if opts[:sort_target]
+      opts[:key] = key
+      Etcdserverpb::RangeRequest.new(opts)
+    end
+
+    def del_request(key, range_end=nil)
+      key = prepend_prefix(@namespace, key)
+      # In order to enforce the scope of the specified namespace, we are going to
+      # intercept the zero-byte reference and re-target everything under the given namespace.
+      if range_end =~ /\x00/
+        range_end = (@namespace[0..-2] + (@namespace[-1].ord + 1).chr)
+      else 
+        range_end = prepend_prefix(@namespace, range_end) if range_end
+      end
+      Etcdserverpb::DeleteRangeRequest.new(key: key, range_end: range_end)
+    end
+
+    def put_request(key, value, lease=nil)
+      key = prepend_prefix(@namespace, key)
+      kv = Etcdserverpb::PutRequest.new(key: key, value: value)
+      kv.lease = lease if lease
+      kv
+    end
+  end
+end

data/lib/etcdv3/namespace/kv/transaction.rb

@@ -0,0 +1,91 @@
+class Etcdv3::Namespace::KV
+  class Transaction
+    include Etcdv3::Namespace::Utilities
+    include Etcdv3::Namespace::KV::Requests
+
+    # Available comparison identifiers.
+    COMPARISON_IDENTIFIERS = {
+      equal: 0,
+      greater: 1,
+      less: 2,
+      not_equal: 3
+    }
+
+    # Available targets to compare with.
+    TARGETS = {
+      version: 0,
+      create_revision: 1,
+      mod_revision: 2,
+      value: 3
+    }
+
+    attr_writer :compare, :success, :failure
+
+    def initialize(namespace)
+      @namespace = namespace
+    end
+
+    def compare
+      @compare ||= []
+    end
+
+    def success
+      @success ||= []
+    end
+
+    def failure
+      @failure ||=[]
+    end
+
+    # Request Operations
+
+    # txn.put('my', 'key', lease_id: 1)
+    def put(key, value, lease=nil)
+      put_request(key, value, lease)
+    end
+
+    # txn.get('key')
+    def get(key, opts={})
+      get_request(key, opts)
+    end
+
+    # txn.del('key')
+    def del(key, range_end='')
+      del_request(key, range_end)
+    end
+
+    ###  Compare Operations
+
+    # txn.version('names', :greater, 0 )
+    def version(key, compare_type, value)
+      generate_compare(:version, key, compare_type, value)
+    end
+
+    # txn.value('names', :equal, 'wowza' )
+    def value(key, compare_type, value)
+      generate_compare(:value, key, compare_type, value)
+    end
+
+    # txn.mod_revision('names', :not_equal, 0)
+    def mod_revision(key, compare_type, value)
+      generate_compare(:mod_revision, key, compare_type, value)
+    end
+
+    # txn.create_revision('names', :less, 10)
+    def create_revision(key, compare_type, value)
+      generate_compare(:create_revision, key, compare_type, value)
+    end
+
+    private
+
+    def generate_compare(target_union, key, compare_type, value)
+      key = prepend_prefix(@namespace, key)    
+      Etcdserverpb::Compare.new(
+        key: key,
+        result: COMPARISON_IDENTIFIERS[compare_type],
+        target: TARGETS[target_union],
+        target_union => value
+      )
+    end
+  end
+end

data/lib/etcdv3/namespace/kv.rb

@@ -0,0 +1,62 @@
+class Etcdv3::Namespace
+  class KV
+    include Etcdv3::Namespace::KV::Requests
+    include Etcdv3::Namespace::Utilities
+    include GRPC::Core::TimeConsts
+  
+    def initialize(hostname, credentials, timeout, namespace, metadata={})
+      @stub = Etcdserverpb::KV::Stub.new(hostname, credentials)
+      @timeout = timeout
+      @namespace = namespace
+      @metadata = metadata
+    end
+
+    def get(key, opts={})
+      timeout = opts.delete(:timeout)
+      resp = @stub.range(get_request(key, opts), metadata: @metadata, deadline: deadline(timeout))
+      strip_prefix(@namespace, resp)
+    end
+
+    def del(key, range_end: '', timeout: nil)
+      resp = @stub.delete_range(del_request(key, range_end), metadata: @metadata, deadline: deadline(timeout))
+      strip_prefix(@namespace, resp)
+    end
+
+    def put(key, value, lease: nil, timeout: nil)
+      resp = @stub.put(put_request(key, value, lease), metadata: @metadata, deadline: deadline(timeout))
+      strip_prefix(@namespace, resp)
+    end
+
+    def transaction(block, timeout: nil)
+      txn = Etcdv3::Namespace::KV::Transaction.new(@namespace)
+      block.call(txn)
+      request = Etcdserverpb::TxnRequest.new(
+        compare: txn.compare,
+        success: generate_request_ops(txn.success),
+        failure: generate_request_ops(txn.failure),
+      )
+      resp = @stub.txn(request, metadata: @metadata, deadline: deadline(timeout))
+      strip_prefix(@namespace, resp)
+    end
+
+    private
+
+    def deadline(timeout)
+      from_relative_time(timeout || @timeout)
+    end
+
+    def generate_request_ops(requests)
+      requests.map do |request|
+        if request.is_a?(Etcdserverpb::RangeRequest)
+          Etcdserverpb::RequestOp.new(request_range: request)
+        elsif request.is_a?(Etcdserverpb::PutRequest)
+          Etcdserverpb::RequestOp.new(request_put: request)
+        elsif request.is_a?(Etcdserverpb::DeleteRangeRequest)
+          Etcdserverpb::RequestOp.new(request_delete_range: request)
+        else
+          raise "Invalid command. Not sure how you got here!"
+        end
+      end
+    end
+  end
+end

data/lib/etcdv3/namespace/lock.rb

@@ -0,0 +1,32 @@
+class Etcdv3::Namespace
+  class Lock
+    include GRPC::Core::TimeConsts
+    include Etcdv3::Namespace::Utilities
+
+    def initialize(hostname, credentials, timeout, namespace, metadata = {})
+      @stub = V3lockpb::Lock::Stub.new(hostname, credentials)
+      @timeout = timeout
+      @namespace = namespace
+      @metadata = metadata
+    end
+
+    def lock(name, lease_id, timeout: nil)
+      name = prepend_prefix(@namespace, name)
+      request = V3lockpb::LockRequest.new(name: name, lease: lease_id)
+      resp = @stub.lock(request, deadline: deadline(timeout))
+      strip_prefix_from_lock(@namespace, resp)
+    end
+
+    def unlock(key, timeout: nil)
+      key = prepend_prefix(@namespace, key)
+      request = V3lockpb::UnlockRequest.new(key: key)
+      @stub.unlock(request, deadline: deadline(timeout))
+    end
+
+    private
+
+    def deadline(timeout)
+      from_relative_time(timeout || @timeout)
+    end
+  end
+end

data/lib/etcdv3/namespace/utilities.rb

@@ -0,0 +1,44 @@
+class Etcdv3::Namespace
+  module Utilities
+    
+    def prepend_prefix(prefix, key)
+      key = key.dup if key.frozen?
+      key.prepend(prefix)
+    end
+
+    def strip_prefix(prefix, resp)
+      [:kvs, :prev_kvs].each do |field|
+        if resp.respond_to?(field)
+          resp.send(field).each do |kv| 
+            kv.key = delete_prefix(prefix, kv.key)
+          end
+        end
+      end
+      resp
+    end
+
+    def strip_prefix_from_lock(prefix, resp)
+      if resp.key
+        resp.key = delete_prefix(prefix, resp.key)
+      end
+      resp 
+    end
+
+    def strip_prefix_from_events(prefix, events)
+      events.each do |event|
+        if event.kv
+          event.kv.key = delete_prefix(prefix, event.kv.key)
+        end
+        if event.prev_kv
+          event.prev_kv.key = delete_prefix(prefix, event.prev_kv.key)
+        end
+        event 
+      end
+    end
+
+    def delete_prefix(prefix, str)
+      str.sub(/\A#{prefix}/, '')
+    end
+
+  end
+end

data/lib/etcdv3/namespace/watch.rb

@@ -0,0 +1,38 @@
+class Etcdv3::Namespace
+  class Watch
+    include GRPC::Core::TimeConsts
+    include Etcdv3::Namespace::Utilities
+
+    def initialize(hostname, credentials, timeout, namespace, metadata = {})
+      @stub = Etcdserverpb::Watch::Stub.new(hostname, credentials)
+      @timeout = timeout
+      @namespace = namespace
+      @metadata = metadata
+    end
+
+    def watch(key, range_end, start_revision, block, timeout: nil)      
+      key = prepend_prefix(@namespace, key)
+      range_end = prepend_prefix(@namespace, range_end) if range_end
+      create_req = Etcdserverpb::WatchCreateRequest.new(key: key)
+      create_req.range_end = range_end if range_end
+      create_req.start_revision = start_revision if start_revision
+      watch_req = Etcdserverpb::WatchRequest.new(create_request: create_req)
+      events = nil
+      @stub.watch([watch_req], metadata: @metadata, deadline: deadline(timeout)).each do |resp|
+        next if resp.events.empty?
+        if block
+          block.call(strip_prefix_from_events(@namespace, resp.events))
+        else
+          events = strip_prefix_from_events(@namespace, resp.events)
+          break
+        end
+      end
+      events
+    end
+
+    def deadline(timeout)
+      from_relative_time(timeout || @timeout)
+    end
+
+  end
+end

data/lib/etcdv3/version.rb

@@ -1,3 +1,3 @@
 class Etcdv3
-  VERSION = '0.9.0'.freeze
+  VERSION = '0.11.4'.freeze
 end

data/lib/etcdv3/watch.rb

@@ -1,16 +1,20 @@
 class Etcdv3
   class Watch
+    include GRPC::Core::TimeConsts
 
-    def initialize(hostname, credentials, _timeout, metadata = {})
+    def initialize(hostname, credentials, timeout, metadata = {})
       @stub = Etcdserverpb::Watch::Stub.new(hostname, credentials)
+      @timeout = timeout
       @metadata = metadata
     end
 
-    def watch(key, range_end, block)
-      create_req = Etcdserverpb::WatchCreateRequest.new(key: key, range_end: range_end)
+    def watch(key, range_end, start_revision, block, timeout: nil)
+      create_req = Etcdserverpb::WatchCreateRequest.new(key: key)
+      create_req.range_end = range_end if range_end
+      create_req.start_revision = start_revision if start_revision
       watch_req = Etcdserverpb::WatchRequest.new(create_request: create_req)
       events = nil
-      @stub.watch([watch_req], metadata: @metadata).each do |resp|
+      @stub.watch([watch_req], metadata: @metadata, deadline: deadline(timeout)).each do |resp|
         next if resp.events.empty?
         if block
           block.call(resp.events)
@@ -21,5 +25,9 @@ class Etcdv3
       end
       events
     end
+
+    def deadline(timeout)
+      from_relative_time(timeout || @timeout)
+    end
   end
 end

data/lib/etcdv3.rb

@@ -7,6 +7,14 @@ require 'etcdv3/auth'
 require 'etcdv3/kv/requests'
 require 'etcdv3/kv/transaction'
 require 'etcdv3/kv'
+
+require 'etcdv3/namespace/utilities'
+require 'etcdv3/namespace/kv/requests'
+require 'etcdv3/namespace/kv/transaction'
+require 'etcdv3/namespace/lock'
+require 'etcdv3/namespace/kv'
+require 'etcdv3/namespace/watch'
+
 require 'etcdv3/maintenance'
 require 'etcdv3/lease'
 require 'etcdv3/watch'
@@ -21,10 +29,16 @@ class Etcdv3
   attr_reader :conn, :credentials, :options
   DEFAULT_TIMEOUT = 120
 
-  def initialize(options = {})
+  def initialize(**options)
     @options = options
     @timeout = options[:command_timeout] || DEFAULT_TIMEOUT
-    @conn = ConnectionWrapper.new(@timeout, *sanitized_endpoints)
+    @namespace = options[:namespace]
+    @conn = ConnectionWrapper.new(
+      @timeout,
+      *sanitized_endpoints,
+      @namespace,
+      @options.fetch(:allow_reconnect, true),
+    )
     warn "WARNING: `url` is deprecated. Please use `endpoints` instead." if @options.key?(:url)
     authenticate(@options[:user], @options[:password]) if @options.key?(:user)
   end
@@ -219,8 +233,8 @@ class Etcdv3
   end
 
   # Watches for changes on a specified key range.
-  def watch(key, range_end: '', &block)
-    @conn.handle(:watch, 'watch', [key, range_end, block])
+  def watch(key, range_end: nil, start_revision: nil, timeout: nil, &block)
+    @conn.handle(:watch, 'watch', [key, range_end, start_revision, block, timeout: timeout])
   end
 
   def transaction(timeout: nil, &block)