ruby_skynet 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fd7fd4822334e7d782d4dea66adfebee81ed5e6
-  data.tar.gz: df2655971d950f5672beadbbe324e866b24ef754
+  metadata.gz: db463c6efbc6b84b670681ac67fd39358dcdcfd1
+  data.tar.gz: 6f18c09efa035cc23435b5a29a8a16c75dcd7437
 SHA512:
-  metadata.gz: c40532f7c7ba9710f1aa48ee5f25fa1a6c1e0a4a1e2fb42d265309abb959e02a0dc1d40c5eada990cb0907898a04806540a1a39d3cd02e1bb2e2065ef12a4325
-  data.tar.gz: 1d877ccc14a6a71bd708161722e45aefbd5b529f9fc62ad6a20f4f7d393d231e4d9f808a849033b8cbfe7a79ecafe0065a65315921c34023c458beaf601a727c
+  metadata.gz: e763b084264b1073831234160c328f9e594d0d33c228240b7487bac01c5fcb9422fd20910a0e3beeca0a38d5cb5f7e1c8521bb49c7751bdbc790bc974c815cb8
+  data.tar.gz: 41976bc92888af6ba2d1e20d75b5babac0d1b5b03b0f38da10489d2e4e9121557bca532c1e9f1ca1f76e8c2adbcc05e6c1af331ae77edd5d0782304bde56bf5f

data/Gemfile.lock

@@ -7,20 +7,18 @@ GEM
       multi_json (~> 1.3)
       thread_safe (~> 0.1)
       tzinfo (~> 0.3.37)
-    atomic (1.1.10)
-    atomic (1.1.10-java)
-    bson (1.9.0)
-    bson (1.9.0-java)
-    bson_ext (1.9.0)
-      bson (~> 1.9.0)
+    atomic (1.1.13)
+    bson (1.9.1)
+    bson_ext (1.9.1)
+      bson (~> 1.9.1)
     gene_pool (1.3.0)
-    i18n (0.6.4)
+    i18n (0.6.5)
     minitest (4.7.5)
-    multi_json (1.7.7)
+    multi_json (1.7.9)
     rake (10.1.0)
     resilient_socket (0.5.0)
       semantic_logger (>= 2.1)
-    ruby_doozer (0.7.1)
+    ruby_doozer (0.8.1)
       gene_pool (>= 1.3.0)
       multi_json (>= 1.6.1)
       resilient_socket (>= 0.5.0)
@@ -28,28 +26,22 @@ GEM
       semantic_logger (>= 2.1)
       sync_attr (>= 1.0.0)
     ruby_protobuf (0.4.11)
-    semantic_logger (2.1.0)
+    semantic_logger (2.2.0)
       sync_attr (>= 1.0)
       thread_safe (>= 0.1.0)
     shoulda (3.5.0)
       shoulda-context (~> 1.0, >= 1.0.1)
       shoulda-matchers (>= 1.4.1, < 3.0)
-    shoulda-context (1.1.4)
-    shoulda-matchers (2.2.0)
+    shoulda-context (1.1.5)
+    shoulda-matchers (2.3.0)
       activesupport (>= 3.0.0)
-    slyphon-log4j (1.2.15)
-    slyphon-zookeeper_jar (3.3.5-java)
     sync_attr (1.0.0)
-    thread_safe (0.1.0)
+    thread_safe (0.1.2)
       atomic
     tzinfo (0.3.37)
     zookeeper (1.4.4)
-    zookeeper (1.4.4-java)
-      slyphon-log4j (= 1.2.15)
-      slyphon-zookeeper_jar (= 3.3.5)
 
 PLATFORMS
-  java
   ruby
 
 DEPENDENCIES

data/lib/ruby_skynet/railtie.rb

@@ -15,8 +15,8 @@ module RubySkynet #:nodoc:
       load "ruby_skynet/railties/ruby_skynet.rake"
     end
 
-    # Load RubySkynet Configuration
-    config.before_configuration do
+    # Load RubySkynet Configuration once rails has started
+    initializer 'ruby_skynet.initialize' do
       config_file = Rails.root.join("config", "ruby_skynet.yml")
       if config_file.file?
         ::RubySkynet.configure!(config_file, Rails.env)

data/lib/ruby_skynet/railties/ruby_skynet.rake

@@ -20,14 +20,14 @@ namespace :ruby_skynet do
 
     RubySkynet::Server.load_services
 
-    # Start the server
-    RubySkynet::Server.start
-    
-    at_exit do
+    begin
+      # Start the server
+      RubySkynet::Server.start
+      RubySkynet::Server.wait_until_server_stops
+    ensure
       RubySkynet::Server.stop
     end
 
-    RubySkynet::Server.wait_until_server_stops
   end
 
 end

data/lib/ruby_skynet/ruby_skynet.rb

@@ -98,10 +98,10 @@ module RubySkynet
 
     # Extract just the zookeeper or doozer configuration element
     RubySkynet.service_registry = ServiceRegistry.new(
-      :registry => config[:registry]
+      :registry => config.delete(:registry)
     )
 
-    config.each_pair {|k,v| RubySkynet::Server.logger.warn "Ignoring unknown RubySkynet config option #{k} => #{v}"}
+    config.each_pair {|k,v| warn "Ignoring unknown RubySkynet config option #{k} => #{v}"}
   end
 
   # Initialize internal class variable

data/lib/ruby_skynet/version.rb

@@ -1,3 +1,3 @@
 module RubySkynet #:nodoc
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/lib/ruby_skynet/zookeeper/registry.rb

@@ -37,6 +37,10 @@ module RubySkynet
       # :ephemeral [Boolean]
       #   All set operations of non-nil values will result in ephemeral nodes.
       #
+      # :on_connect [Proc]
+      #   Block to call after the connection to Zookeeper has been established
+      #   and every time the connection is re-established
+      #
       # :registry [Hash|ZooKeeper]
       #   ZooKeeper configuration information, or an existing
       #   ZooKeeper ( ZooKeeper client) instance
@@ -72,19 +76,13 @@ module RubySkynet
         @root = '/' if @root == ''
 
         registry_config = params.delete(:registry) || {}
-        if registry_config.is_a?(::Zookeeper::Client)
-          @zookeeper = registry_config
-        else
-          servers = registry_config.delete(:servers) || ['127.0.0.1:2181']
-          connect_timeout = (registry_config.delete(:connect_timeout) || 10).to_f
 
-          # Generate warning log entries for any unknown configuration options
-          registry_config.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: zookeeper.#{k}"}
+        #   server1:2181,server2:2181,server3:2181
+        @servers = (registry_config.delete(:servers) || ['127.0.0.1:2181']).join(',')
+        @connect_timeout = (registry_config.delete(:connect_timeout) || 10).to_f
 
-          # Create Zookeeper connection
-          #   server1:2181,server2:2181,server3:2181
-          @zookeeper = ::Zookeeper.new(servers.join(','), connect_timeout, watcher)
-        end
+        # Generate warning log entries for any unknown configuration options
+        registry_config.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: zookeeper.#{k}"}
 
         # Allow the serializer and deserializer implementations to be replaced
         @serializer   = params.delete(:serializer)   || RubySkynet::Zookeeper::Json::Serializer
@@ -93,18 +91,18 @@ module RubySkynet
         @ephemeral = params.delete(:ephemeral)
         @ephemeral = false if @ephemeral.nil?
 
+        @on_connect = params.delete(:on_connect)
+
         # Generate warning log entries for any unknown configuration options
         params.each_pair {|k,v| logger.warn "Ignoring unknown configuration option: #{k}"}
 
         # Hash with Array values containing the list of children for each node, if any
         @children = ThreadSafe::Hash.new
 
-        # Start watching registry for any changes
-        get_recursive(@root, watch=true, create_path=true, &block)
+        # Block is used in init
+        @block = block
 
-        at_exit do
-          close
-        end
+        self.init
       end
 
       # Retrieve the latest value from a specific path from the registry
@@ -326,69 +324,83 @@ module RubySkynet
         end
       end
 
-      # returns the watcher proc for this registry instances
+      # returns the watcher proc for this registry instance
       def watcher
         # Subscription block to call for watch events
         @watch_proc ||= Proc.new do |event_hash|
-          path = event_hash[:path]
-          case event_hash[:type]
-          when ::Zookeeper::ZOO_CHANGED_EVENT
-            logger.debug "Node '#{path}' Changed", event_hash
-
-            # Fetch current value and re-subscribe
-            result = @zookeeper.get(:path => path, :watcher => @watch_proc)
-            check_rc(result)
-            value = @deserializer.deserialize(result[:data])
-            stat = result[:stat]
-
-            # Invoke on_update callbacks
-            node_updated(relative_key(path), value, stat.version)
-
-          when ::Zookeeper::ZOO_DELETED_EVENT
-            # A node has been deleted
-            # TODO How to ignore child deleted when it is a directory, not a leaf
-            logger.debug "Node '#{path}' Deleted", event_hash
-            @children.delete(path)
-            node_deleted(relative_key(path))
-
-          when ::Zookeeper::ZOO_CHILD_EVENT
-            # The list of nodes has changed - Does not say if it was added or removed
-            logger.debug "Node '#{path}' Child changed", event_hash
-            result = @zookeeper.get_children(:path => path, :watcher => @watch_proc)
-
-            # This node could have been deleted already
-            if result[:rc] == ::Zookeeper::ZOK
-              current_children = result[:children]
-              previous_children = @children[path]
-
-              # Save children so that we can later identify new children
-              @children[path] = current_children
-
-              # New Child Nodes
-              new_nodes = previous_children ? (current_children - previous_children) : current_children
-              new_nodes.each do |child|
-                get_recursive(File.join(path,child), true) do |key, value, version|
-                  node_created(key, value, version)
+          begin
+            path = event_hash[:path]
+            logger.trace "Event Received", event_hash
+            case event_hash[:type]
+            when ::Zookeeper::ZOO_CHANGED_EVENT
+              logger.debug "Node '#{path}' Changed", event_hash
+
+              # Fetch current value and re-subscribe
+              result = @zookeeper.get(:path => path, :watcher => @watch_proc)
+              check_rc(result)
+              value = @deserializer.deserialize(result[:data])
+              stat = result[:stat]
+
+              # Invoke on_update callbacks
+              node_updated(relative_key(path), value, stat.version)
+
+            when ::Zookeeper::ZOO_DELETED_EVENT
+              # A node has been deleted
+              # TODO How to ignore child deleted when it is a directory, not a leaf
+              logger.debug "Node '#{path}' Deleted", event_hash
+              @children.delete(path)
+              node_deleted(relative_key(path))
+
+            when ::Zookeeper::ZOO_CHILD_EVENT
+              # The list of nodes has changed - Does not say if it was added or removed
+              logger.debug "Node '#{path}' Child changed", event_hash
+              result = @zookeeper.get_children(:path => path, :watcher => @watch_proc)
+
+              # This node could have been deleted already
+              if result[:rc] == ::Zookeeper::ZOK
+                current_children = result[:children]
+                previous_children = @children[path]
+
+                # Save children so that we can later identify new children
+                @children[path] = current_children
+
+                # New Child Nodes
+                new_nodes = previous_children ? (current_children - previous_children) : current_children
+                new_nodes.each do |child|
+                  get_recursive(File.join(path,child), true) do |key, value, version|
+                    node_created(key, value, version)
+                  end
                 end
+                # Ignore Deleted Child Nodes since they will be handled by the Deleted Node event
               end
-              # Ignore Deleted Child Nodes since they will be handled by the Deleted Node event
-            end
 
-          when ::Zookeeper::ZOO_CREATED_EVENT
-            # Node created events are only created for paths that were deleted
-            # and then created again
-            # No op - This is covered by node_child created event
-            logger.debug "Node '#{path}' Created - No op", event_hash
+            when ::Zookeeper::ZOO_CREATED_EVENT
+              # Node created events are only created for paths that were deleted
+              # and then created again
+              # No op - This is covered by node_child created event
+              logger.debug "Node '#{path}' Created - No op", event_hash
+
+            when ::Zookeeper::ZOO_SESSION_EVENT
+              logger.debug "Session Event: #{@zookeeper.state_by_value(event_hash[:state]) if @zookeeper}", event_hash
 
-          when ::Zookeeper::ZOO_SESSION_EVENT
-            logger.debug "Session Event: #{@zookeeper.state_by_value(event_hash[:state])}", event_hash
+              # Replace zookeeper connection since it is stale. Only react to global request
+              # since this event will be received for every node being watched.
+              #   Do not close the current connection since this background watcher thread is running
+              #   as part of the current zookeeper connection
+              #     event_hash => {:req_id=>-1, :type=>-1, :state=>-112, :path=>"", :context=>nil}
+              Thread.new { self.init } if (event_hash[:req_id] == -1) && (event_hash[:state] == ::Zookeeper::ZOO_EXPIRED_SESSION_STATE)
 
-          when ::Zookeeper::ZOO_NOTWATCHING_EVENT
-            logger.debug "Ignoring ZOO_NOTWATCHING_EVENT", event_hash
+            when ::Zookeeper::ZOO_NOTWATCHING_EVENT
+              logger.debug "Ignoring ZOO_NOTWATCHING_EVENT", event_hash
 
-          else
-            # TODO Need to re-load registry when re-connected
-            logger.warn "Ignoring unknown event", event_hash
+            else
+              # TODO Need to re-load registry when re-connected
+              logger.warn "Ignoring unknown event", event_hash
+            end
+          rescue ::Zookeeper::Exceptions::ZookeeperException => exc
+            logger.warn "Watching thread failed due to Zookeeper failure", exc
+          rescue Exception => exc
+            logger.error "Watching thread failed due to unhandled exception", exc
           end
         end
       end
@@ -511,6 +523,24 @@ module RubySkynet
         end
       end
 
+      # Create ZooKeeper connection and start watching the registry for any changes
+      def init
+        logger.benchmark_info "Connected to Zookeeper" do
+          @zookeeper.close if @zookeeper
+          # Create Zookeeper connection
+          @zookeeper = ::Zookeeper.new(@servers, @connect_timeout, watcher)
+          at_exit do
+            @zookeeper.close if @zookeeper
+          end
+
+          # Start watching registry for any changes
+          get_recursive(@root, watch=true, create_path=true, &@block)
+
+          # Call on_connect callback if supplied
+          @on_connect.call(self) if @on_connect
+        end
+      end
+
     end
   end
 end

data/lib/ruby_skynet/zookeeper/service_registry.rb

@@ -23,9 +23,17 @@ module RubySkynet
         @cache = ThreadSafe::Hash.new
         @notifications_cache = ThreadSafe::Hash.new
 
+        # Keep a list of registered services so that they can be re-registered
+        # if the connection is lost
+        @services = ThreadSafe::Hash.new
+
         # Supply block to load the current keys from the Registry
         params[:root] = '/instances'
         params[:ephemeral] = true
+        params[:on_connect] = Proc.new do |registry|
+          # Re-Register services every time the connection to ZooKeeper is lost
+          @services.values.each {|v| register_service(*v)}
+        end
         @registry = Zookeeper::Registry.new(params) do |key, value|
           service_info_created(key, value)
         end
@@ -44,13 +52,14 @@ module RubySkynet
       # Returns the UUID for the service that was created
       def register_service(name, version, region, hostname, port)
         uuid = "#{hostname}:#{port}-#{$$}-#{name}-#{version}"
-        # TODO Make sets ephemeral
         @registry[File.join(uuid,'addr')]       = "#{hostname}:#{port}"
         @registry[File.join(uuid,'name')]       = name
         @registry[File.join(uuid,'version')]    = version
         @registry[File.join(uuid,'region')]     = region
         @registry[File.join(uuid,'registered')] = true
-	      uuid
+        # Add to local services list
+        @services[uuid] = [name, version, region, hostname, port]
+        uuid
       end
 
       # Deregister the supplied service from the Registry
@@ -62,6 +71,9 @@ module RubySkynet
         @registry.delete(File.join(uuid,'region'), false)
         @registry.delete(File.join(uuid,'registered'), false)
         @registry.delete(uuid, false)
+        # Remove from local services list
+        @services.delete(uuid)
+	      uuid
       end
 
       # Return a server that implements the specified service
@@ -105,21 +117,21 @@ module RubySkynet
       protected
 
       def service_info_deleted(path)
-        logger.info("service_info_deleted: #{path}")
+        logger.trace("service_info_deleted: #{path}")
         # path: "uuid/key"
         uuid, key = path.split('/')
-        if (key == 'registered')
-          if server = @notifications_cache.delete(uuid)
-            hostname, port = server['addr'].split(':')
-            # Service has stopped and needs to be removed
-            remove_server(File.join(server['name'], server['version'].to_s, server['region']), hostname, port, true)
-          end
+        # If any child of the node is deleted, deregister the service,
+        # not just for (key == 'registered')
+        if server = @notifications_cache.delete(uuid)
+          hostname, port = server['addr'].split(':')
+          # Service has stopped and needs to be removed
+          remove_server(File.join(server['name'], server['version'].to_s, server['region']), hostname, port, true)
         end
       end
 
       # Service information created
       def service_info_created(path, value=nil)
-        logger.info("service_info_created: #{path}", value)
+        logger.trace("service_info_created: #{path}", value)
         # path: "uuid/key"
         uuid, key = path.split('/')
 
@@ -135,7 +147,7 @@ module RubySkynet
 
       # Service information changed
       def service_info_updated(path, value=nil)
-        logger.info("service_info_updated: #{path}", value)
+        logger.trace("service_info_updated: #{path}", value)
         # path: "uuid/key"
         uuid, key = path.split('/')
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_skynet
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Reid Morrison
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-07 00:00:00.000000000 Z
+date: 2013-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: semantic_logger