ruby_doozer 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3820705ac3d8550a61d136be5b7134c806dab86e
-  data.tar.gz: 35612961ac704ed090141234b65ab42490f1998c
+  metadata.gz: ca8f8f8ae259ceb2da7979c8ca5c7a2307b6fcc7
+  data.tar.gz: 4faf064cc2e33a25a3e612f66d59c528c948e491
 SHA512:
-  metadata.gz: 86f04267b25ba51ef9c7a935b45b41265c8faa124df655d7d10c626e93bc3d462637ab6a6b5f0d7c85678ce935a1e995897379c07ef723f5476daaafbe430ff2
-  data.tar.gz: 7890530ad60348d8483ee03208b67a10a89a76e86144618742ab0dccd09399bb23aad84fd2a5ae8357bef7d83bf56a8e32c072ee9ada47a50c8241a475531793
+  metadata.gz: 2f085c6219b6a3901d37937079036df515e5ad9fe4e54253147109e10cb3d8b51fa74cab823336883f7ad0d049d53149474f8d1fa31384a167137be7574b8a2f
+  data.tar.gz: 1a3c63775c7587c24f984b5d130c9aa17c464d338c305434df7f4127a472a154891f229f4639f96728b538db3bad9f89b55d9d44eba4d5cb7984d2f275d602fa

data/Gemfile

@@ -2,10 +2,10 @@ source :rubygems
 
 group :test do
   gem "shoulda"
-  gem "mocha", :require => false
 end
 
 gem "rake"
 gem "semantic_logger"
 gem "resilient_socket"
 gem "ruby_protobuf"
+gem "gene_pool"

data/Gemfile.lock

@@ -7,6 +7,7 @@ GEM
     atomic (1.0.1)
     bourne (1.4.0)
       mocha (~> 0.13.2)
+    gene_pool (1.3.0)
     i18n (0.6.1)
     metaclass (0.0.1)
     mocha (0.13.3)
@@ -34,7 +35,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  mocha
+  gene_pool
   rake
   resilient_socket
   ruby_protobuf

data/README.md

@@ -19,22 +19,62 @@ client.close
 
 ### Logging
 
-Since ruby_skynet uses SemanticLogger, trace level logging of all TCP/IP
+Since ruby_doozer uses SemanticLogger, trace level logging of all TCP/IP
 calls can be enabled as follows:
 
 ```ruby
 require 'rubygems'
-require 'ruby_skynet'
+require 'ruby_doozer'
 
 SemanticLogger::Logger.default_level = :trace
-SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('skynet.log')
+SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('doozer.log')
+
+client = RubyDoozer::Client.new(:server => '127.0.0.1:8046')
+client.set('/test/foo', 'value')
+result = client.get('/test/foo')
+client.close
+```
+
+### Registry
+
+RubyDoozer also includes a Registry class to support storing all configuration
+information in doozer. This Centralized Configuration allows configuration changes
+to be made dynamically at run-time and all interested parties will be notified
+of the changes.
+
+For example, making a change to the central database configuration will notify
+all application servers to drop their database connections and re-establish them
+to the new servers:
+
+```ruby
+require 'rubygems'
+require 'ruby_doozer'
+
+SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('registry.log')
+
+region      = "Development"
+application = "sprites"
+path        = "/#{region}/#{application}/config/resources".downcase
+
+config_registry = RubyDoozer::Registry.new(:root_path => path)
+
+# Store the configuration information in doozer as a serialized string
+config_registry['master'] = "Some JSON config string"
+
+# Allow time for Doozer to publish the new config
+sleep 0.5
+
+# Retrieve the current configuration
+database_config = config_registry['master']
+slave_database_config = config_registry['secondary']
 
-class EchoService
-  include RubySkynet::Base
+# Register for any changes to the configuration
+config_registry.on_update('master') do |path, value|
+  puts "Time to re-establish database connections to new server: #{value}"
 end
 
-client = EchoService.new
-p client.echo(:hello => 'world')
+# Change the configuration and all subscribers will be notified
+config_registry['master'] = "Some updated JSON config string"
 ```
 
 ### Notes

data/Rakefile

@@ -6,7 +6,6 @@ require 'rubygems/package'
 require 'rake/clean'
 require 'rake/testtask'
 require 'date'
-require 'semantic_logger'
 require 'ruby_doozer/version'
 
 desc "Build gem"
@@ -27,6 +26,7 @@ task :gem  do |t|
     spec.add_dependency 'semantic_logger'
     spec.add_dependency 'resilient_socket'
     spec.add_dependency 'ruby_protobuf'
+    spec.add_dependency 'gene_pool'
   end
   Gem::Package.build gemspec
 end

data/lib/ruby_doozer.rb

@@ -1,3 +1,6 @@
+require 'semantic_logger'
+
 module RubyDoozer
   autoload :Client,   'ruby_doozer/client'
+  autoload :Registry, 'ruby_doozer/registry'
 end

data/lib/ruby_doozer/registry.rb

@@ -0,0 +1,379 @@
+require 'thread_safe'
+require 'gene_pool'
+require 'semantic_logger'
+
+#
+# Registry
+#
+# Store information in doozer and subscribe to future changes
+#
+# Notifies registered subscribers when information has changed
+#
+# All paths specified are relative to the root_path. As such the root path
+# is never returned, nor is it required when a path is supplied as input.
+# For example, with a root_path of /foo/bar, any paths passed in will leave
+# out the root_path: host/name
+#
+module RubyDoozer
+  class Registry
+    # Logging instance for this class
+    include SemanticLogger::Loggable
+
+    attr_reader :doozer_config, :doozer_pool
+
+    # Create a Registry instance to manage a path of information within doozer
+    #
+    # :root_path [String]
+    #   Root path to load and then monitor for changes
+    #   It is not recommended to set the root_path to "/" as it will generate
+    #   significant traffic since it will also monitor Doozer Admin changes
+    #   Mandatory
+    #
+    # :cache [Boolean]
+    #   Keep a local copy in memory of all descendant values of the supplied root_path
+    #   Allows high-frequency calls to retrieve registry data
+    #   The cache will be kept in synch with any changes on the server
+    #   Default: true
+    #
+    # :doozer [Hash]
+    #   Doozer configuration information
+    #
+    #   :servers [Array of String]
+    #     Array of URL's of doozer servers to connect to with port numbers
+    #     ['server1:2000', 'server2:2000']
+    #
+    #     An attempt will be made to connect to alternative servers when the
+    #     current server cannot be connected to
+    #     Default: ['127.0.0.1:8046']
+    #
+    #   :read_timeout [Float]
+    #     Time in seconds to timeout on read
+    #     Can be overridden by supplying a timeout in the read call
+    #     Default: 5
+    #
+    #   :connect_timeout [Float]
+    #     Time in seconds to timeout when trying to connect to the server
+    #     Default: 3
+    #
+    #   :connect_retry_count [Fixnum]
+    #     Number of times to retry connecting when a connection fails
+    #     Default: 10
+    #
+    #   :connect_retry_interval [Float]
+    #     Number of seconds between connection retry attempts after the first failed attempt
+    #     Default: 0.5
+    #
+    #   :server_selector [Symbol|Proc]
+    #     When multiple servers are supplied using :servers, this option will
+    #     determine which server is selected from the list
+    #       :ordered
+    #         Select a server in the order supplied in the array, with the first
+    #         having the highest priority. The second server will only be connected
+    #         to if the first server is unreachable
+    #       :random
+    #         Randomly select a server from the list every time a connection
+    #         is established, including during automatic connection recovery.
+    #       Proc:
+    #         When a Proc is supplied, it will be called passing in the list
+    #         of servers. The Proc must return one server name
+    #           Example:
+    #             :server_selector => Proc.new do |servers|
+    #               servers.last
+    #             end
+    #       Default: :random
+    #
+    #   :pool_size [Integer]
+    #     Maximum size of the connection pool to doozer
+    #     Default: 10
+    #
+    def initialize(params)
+      params = params.dup
+      @root_path = params.delete(:root_path)
+      raise "Missing mandatory parameter :root_path" unless @root_path
+
+      # Add leading '/' to root_path if missing
+      @root_path = "/#{@root_path}" unless @root_path.start_with?('/')
+
+      # Strip trailing '/' if supplied
+      @root_path = @root_path[0..-2] if @root_path.end_with?("/")
+      @root_path_with_trail = "#{@root_path}/"
+
+      local_copy = params.delete(:cache)
+      @registry = (local_copy == false) ? nil : ThreadSafe::Hash.new
+
+      @doozer_config = params.delete(:doozer) || {}
+      @doozer_config[:servers]                ||= ['127.0.0.1:8046']
+      @doozer_config[:read_timeout]           ||= 5
+      @doozer_config[:connect_timeout]        ||= 3
+      @doozer_config[:connect_retry_interval] ||= 0.5
+      @doozer_config[:connect_retry_count]    ||= 10
+      @doozer_config[:server_selector]        ||= :random
+
+      # Connection pool settings
+      @doozer_pool = GenePool.new(
+        :name         =>"Doozer Connection Pool",
+        :pool_size    => @doozer_config.delete(:pool_size) || 10,
+        :timeout      => @doozer_config.delete(:pool_timeout) || 30,
+        :warn_timeout => @doozer_config.delete(:pool_warn_timeout) || 5,
+        :idle_timeout => @doozer_config.delete(:pool_idle_timeout) || 600,
+        :logger       => logger,
+        :close_proc   => :close
+      ) do
+        RubyDoozer::Client.new(@doozer_config)
+      end
+
+      @create_subscribers = ThreadSafe::Hash.new
+      @update_subscribers = ThreadSafe::Hash.new
+      @delete_subscribers = ThreadSafe::Hash.new
+
+      revision = nil
+      path = "#{@root_path}/**"
+      doozer_pool.with_connection do |doozer|
+        revision = doozer.current_revision
+        if @registry
+          # Fetch all the configuration information from Doozer and set the internal copy
+          doozer.walk(path, revision).each do |node|
+            @registry[relative_path(node.path)] = node.value
+          end
+        end
+      end
+
+      # Start monitoring thread to keep the registry in synch with doozer
+      @monitor_thread = Thread.new { watch_registry(path, revision + 1) }
+
+      # Generate warning log entries for any unknown configuration options
+      params.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: #{k}"}
+    end
+
+    # Retrieve the latest value from a specific path from the registry
+    # If :cache was set to false on the initializer this call will
+    # make a network call to doozer to retrieve the current value
+    # Otherwise it is an in memory call and can be called frequently
+    def [](path)
+      if @registry
+        @registry[path]
+      else
+        doozer_pool.with_connection do |doozer|
+          doozer[full_path(path)]
+        end
+      end
+    end
+
+    # Replace the latest value at a specific path
+    # The in-memory copy will be updated when doozer sends out the change
+    # TODO Should we also update the in memory copy rather than wait for the change
+    #      notification?
+    def []=(path,value)
+      doozer_pool.with_connection do |doozer|
+        doozer[full_path(path)] = value
+      end
+    end
+
+    # Delete the value at a specific path
+    # The in-memory copy will be updated when doozer sends out the change
+    # TODO Should we also delete the in memory copy rather than wait for the change
+    #      notification?
+    def delete(path)
+      old_value = @registry[path] if @registry
+      doozer_pool.with_connection do |doozer|
+        doozer.delete(full_path(path))
+      end
+      old_value
+    end
+
+    # Iterate over every key, value pair in the registry at the root_path
+    #
+    # If :cache was set to false on the initializer this call will
+    # make network calls to doozer to retrieve the current values
+    # Otherwise it is an in memory call against a duplicate of the registry
+    #
+    # Example:
+    #   registry.each_pair {|k,v| puts "#{k} => #{v}"}
+    def each_pair(&block)
+      if @registry
+        @registry.dup.each_pair(&block)
+      else
+        path = "#{@root_path}/**"
+        doozer_pool.with_connection do |doozer|
+          doozer.walk(path, revision).each do |node|
+            block.call(relative_path(node.path), node.value)
+          end
+        end
+      end
+    end
+
+    # Returns a copy of the registry as a Hash
+    def to_h
+      if @registry
+        @registry.dup
+      else
+        h = {}
+        each_pair {|k,v| h[k] = v}
+        h
+      end
+    end
+
+    # Cleanup on process termination
+    def finalize
+      logger.info "Finalizing"
+      if @monitor_thread
+        @monitor_thread.kill
+        @monitor_thread = nil
+      end
+      @doozer_pool.close if @doozer_pool
+      @doozer_pool = nil
+    end
+
+    # When an entry is created the block will be called
+    #  Parameters
+    #    path
+    #      The relative path _excluding_ the root_path to watch for changes
+    #      #TODO Or a regular expression
+    #    block
+    #      The block to be called
+    #
+    #  Parameters passed to the block:
+    #    path
+    #      The path that was created
+    #
+    #    value
+    #      New value from doozer
+    #
+    # Example:
+    #   registry.on_update do |path, value|
+    #     puts "#{path} was created with #{value}"
+    #   end
+    def on_create(path, &block)
+      (@create_subscribers[path] ||= ThreadSafe::Array.new) << block
+    end
+
+    # When an entry is updated the block will be called
+    #  Parameters
+    #    path
+    #      The relative path _excluding_ the root_path to watch for changes
+    #    block
+    #      The block to be called
+    #
+    #  Parameters passed to the block:
+    #    path
+    #      The path that was deleted from doozer
+    #
+    #    value
+    #      New value from doozer
+    #
+    #    old_value
+    #      nil if :cache is false
+    #      otherwise the last value for this path in the local copy of the registry
+    #
+    # Example:
+    #   registry.on_update do |path, value, old_value|
+    #     puts "#{path} was updated to #{value} with previous value #{old_value}"
+    #   end
+    #
+    # Warning:
+    #   If :cache is set to false then on_update will never be called
+    #   Every change will go to subscribers of #on_create()
+    def on_update(path, &block)
+      (@update_subscribers[path] ||= ThreadSafe::Array.new) << block
+    end
+
+    # When an entry is deleted the block will be called
+    #  Parameters
+    #    path
+    #      The relative path _excluding_ the root_path to watch for changes
+    #    block
+    #      The block to be called
+    #
+    #  Parameters passed to the block:
+    #    path
+    #      The path that was deleted from doozer
+    #
+    #    old_value
+    #      nil if :cache is false
+    #      otherwise the last value for this path in the local copy of the registry
+    #
+    # Example:
+    #   registry.on_delete do |path, old_value|
+    #     puts "#{path} was deleted with previous value #{old_value}"
+    #   end
+    def on_delete(path, &block)
+      (@delete_subscribers[path] ||= ThreadSafe::Array.new) << block
+    end
+
+    ############################
+    protected
+
+    #Subscription = Struct.new(:path, :block, :reg_exp)
+
+
+    # Returns the full path given a relative path
+    def full_path(relative_path)
+      "#{@root_path}/#{relative_path}"
+    end
+
+    # Returns the full path given a relative path
+    def relative_path(full_path)
+      full_path.sub(@root_path_with_trail, '')
+    end
+
+    # The path has been added or updated in the registry
+    def changed(path, value)
+      logger.debug { "Changed: #{path} => #{value}" }
+      old_value = nil
+
+      # Keeping a local copy of the registry?
+      if @registry
+        # Already have a value?
+        old_value = @registry[path]
+
+        # Update in memory copy
+        @registry[path] = value
+      end
+
+      if subscribers = old_value ? @update_subscribers[path] : @create_subscribers[path]
+        subscribers.each{|subscriber| subscriber.call(path, value, old_value)}
+      end
+    end
+
+    # Existing data has been removed from the registry
+    def deleted(path)
+      logger.debug { "Deleted: #{path}" }
+      old_value = @registry.delete(path) if @registry
+      if subscribers = @delete_subscribers[path]
+        subscribers.each{|subscriber| subscriber.call(path, old_value)}
+      end
+    end
+
+    # Waits for any updates from Doozer and updates the internal service registry
+    def watch_registry(watch_path, revision)
+      logger.info "Start monitoring #{watch_path}"
+      # This thread must use its own dedicated doozer connection
+      doozer = RubyDoozer::Client.new(@doozer_config)
+
+      # Watch for any changes
+      doozer.watch(watch_path, revision) do |node|
+        logger.trace "Registry Change Notification", node
+
+        # Remove the Root path
+        path = relative_path(node.path)
+
+        case node.flags
+        when 4
+          changed(path, node.value)
+        when 8
+          deleted(path)
+        else
+          logger.error "Unknown flags returned by doozer:#{node.flags}"
+        end
+        logger.trace "Updated registry", @registry
+      end
+      logger.info "Stopping monitoring thread normally"
+    rescue Exception => exc
+      logger.error "Exception in monitoring thread", exc
+    ensure
+      doozer.close if doozer
+      logger.info "Stopped monitoring for changes in the doozer registry"
+    end
+
+  end
+end

data/lib/ruby_doozer/version.rb

@@ -1,3 +1,3 @@
 module RubyDoozer #:nodoc
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/test/client_test.rb

@@ -9,6 +9,12 @@ require 'ruby_doozer/client'
 # NOTE:
 # This test assumes that doozerd is running locally on the default port of 8046
 
+# Register an appender if one is not already registered
+if SemanticLogger::Logger.appenders.size == 0
+  SemanticLogger::Logger.default_level = :trace
+  SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('test.log')
+end
+
 # Unit Test for RubyDoozer::Client
 class ClientTest < Test::Unit::TestCase
   context RubyDoozer::Client do

data/test/registry_test.rb

@@ -0,0 +1,120 @@
+# Allow test to be run in-place without requiring a gem install
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require 'ruby_doozer'
+
+# NOTE:
+# This test assumes that doozerd is running locally on the default port of 8046
+
+# Register an appender if one is not already registered
+if SemanticLogger::Logger.appenders.size == 0
+  SemanticLogger::Logger.default_level = :trace
+  SemanticLogger::Logger.appenders << SemanticLogger::Appender::File.new('test.log')
+end
+
+# Unit Test for RubyDoozer::Client
+class RegistryTest < Test::Unit::TestCase
+  context RubyDoozer::Registry do
+    context "with test data" do
+      setup do
+        @test_data = {
+          'bar' => 'test',
+          'one' => 'one',
+          'two' => 'two',
+        }
+        # Doozer does not allow '_' in path names
+        @root_path = "/registrytest"
+        @client = RubyDoozer::Client.new(:server => 'localhost:8046')
+        @test_data.each_pair {|k,v| @client.set("#{@root_path}/#{k}",v)}
+
+        @registry = RubyDoozer::Registry.new(:root_path => @root_path)
+      end
+
+      def teardown
+        @registry.finalize if @registry
+        if @client
+          @test_data.each_pair do |k,v|
+            @client.delete("#{@root_path}/#{k}")
+          end
+          @client.delete("#{@root_path}/three")
+          @client.close
+        end
+      end
+
+      # Run tests with and without a local cache
+      [true, false].each do |cache|
+        context "cache:#{cache}" do
+          should "have complete registry" do
+            @test_data.each_pair do |k,v|
+              assert_equal v, @registry[k], "Expected #{k}=>#{v}, #{@registry.to_h.inspect}"
+            end
+          end
+
+          should "iterate over complete registry" do
+            @registry.each_pair do |k,v|
+              assert_equal v, @test_data[k], "Registry #{k}=>#{v}, #{@registry.to_h.inspect}"
+            end
+          end
+
+          should "successfully set and retrieve data" do
+            @registry['three'] = 'value'
+            # Allow doozer to send back the change
+            sleep 0.5
+            result = @registry['three']
+            assert_equal 'value', result
+          end
+
+          should "invoke callbacks on create" do
+            created_path = nil
+            created_value = nil
+            @registry.on_create('three') do |path, value|
+              created_path = path
+              created_value = value
+            end
+            @registry['three'] = 'created'
+            # Allow doozer to send back the change
+            sleep 0.5
+            assert_equal 'three', created_path
+            assert_equal 'created', created_value
+          end
+
+          should "invoke callbacks on update" do
+            # Update only triggers when the cache is enabled
+            if cache
+              updated_path = nil
+              updated_value = nil
+              @registry.on_update('bar') do |path, value|
+                updated_path = path
+                updated_value = value
+              end
+              @registry['bar'] = 'updated'
+              # Allow doozer to send back the change
+              sleep 0.5
+              assert_equal 'bar', updated_path
+              assert_equal 'updated', updated_value
+            end
+          end
+
+          should "invoke callbacks on delete" do
+            deleted_path = nil
+            deleted_value = nil
+            @registry.on_delete('bar') do |path, old_value|
+              deleted_path = path
+              deleted_value = old_value
+            end
+            # Allow doozer to send back the change
+            @registry.delete('bar')
+            sleep 0.5
+            assert_equal 'bar', deleted_path
+            assert_equal 'test', deleted_value
+          end
+
+        end
+      end
+
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_doozer
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-28 00:00:00.000000000 Z
+date: 2013-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: semantic_logger
@@ -52,6 +52,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: gene_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby Client for doozer
 email:
 - reidmo@gmail.com
@@ -68,8 +82,10 @@ files:
 - lib/ruby_doozer/client.rb
 - lib/ruby_doozer/exceptions.rb
 - lib/ruby_doozer/msg.pb.rb
+- lib/ruby_doozer/registry.rb
 - lib/ruby_doozer/version.rb
 - test/client_test.rb
+- test/registry_test.rb
 homepage: https://github.com/ClarityServices/ruby_doozer
 licenses:
 - Apache License V2.0