hoodoo 1.0.2

data/lib/hoodoo/services/discovery/discoverers/by_drb/by_drb.rb

@@ -0,0 +1,195 @@
+########################################################################
+# File::    by_drb.rb
+# (C)::     Loyalty New Zealand 2015
+#
+# Purpose:: Discover resource endpoint locations via a DRb registry. For
+#           HTTP-based endpoints.
+# ----------------------------------------------------------------------
+#           03-Mar-2015 (ADH): Created.
+########################################################################
+
+require 'drb/drb'
+
+module Hoodoo
+  module Services
+    class Discovery # Just used as a namespace here
+
+      # Discover resource endpoint locations via a DRb registry. For
+      # HTTP-based endpoints.
+      #
+      class ByDRb < Hoodoo::Services::Discovery
+
+        public
+
+          # Intended for testing only - flushes the records held in the
+          # DRb service.
+          #
+          def flush_services_for_test
+            drb_service().flush()
+          end
+
+        protected
+
+          # Configure an instance. Call via
+          # Hoodoo::Services::Discovery::Base#new. Parameters:
+          #
+          # +options+:: Options hash as described below.
+          #
+          # Options are:
+          #
+          # +drb_port+:: Optional port on which to launch the DRb service.
+          #              If omitted, environment variable
+          #              +HOODOO_DISCOVERY_BY_DRB_PORT_OVERRIDE+ will be
+          #              consulted. If unset, port 8787 is used.
+          #
+          # +drb_uri+::  Optional URI String at which to find an existing DRB
+          #              service. It must alreayd be running. If omitted, the
+          #              +drb_port+ option's behaviour applies. If present,
+          #              the +drb_port+ option is ignored.
+          #
+          def configure_with( options )
+            @drb_port = options[ :drb_port ]
+            @drb_uri  = options[ :drb_uri  ]
+          end
+
+          # Announce the location of an instance through the DRb service
+          # (which may be started up if necessary).
+          #
+          # Returns a Hoodoo::Services::Discovery::ForHTTP instance.
+          #
+          # Call via Hoodoo::Services::Discovery::Base#announce.
+          #
+          # +resource+:: Passed to #discover_remote.
+          # +version+::  Passed to #discover_remote.
+          # +options+::  Options hash as described below.
+          #
+          # Options keys are currently all required:
+          #
+          # +host+:: Host name as a string for location of service endpoint,
+          #          over HTTP (usually, local development is assumed).
+          #
+          # +port+:: Port number of service endpoint.
+          #
+          # +path+:: Path on the above host and port of service endpoint.
+          #
+          def announce_remote( resource, version, options = {} )
+
+            host = options[ :host ]
+            port = options[ :port ]
+            path = options[ :path ]
+
+            endpoint_uri_string = "http://#{ host }:#{ port }#{ path }"
+
+            # Announce our local services if we managed to find the host and port,
+            # but no point otherwise; the values could be anything. In a 'guard'
+            # based environment, first-run determines host and port but subsequent
+            # runs do not - yet it stays the same, so it works out OK there.
+            #
+            unless host.nil? || port.nil? || discover_remote( resource, version )
+              drb_service().add( resource, version, endpoint_uri_string )
+            end
+
+            return result_for( resource, version, endpoint_uri_string )
+          end
+
+          # Discover an endpoint someone previously registered via
+          # #announce_remote.
+          #
+          # Returns a Hoodoo::Services::Discovery::ForHTTP instance if
+          # the endpoint is found, else +nil+.
+          #
+          # +resource+:: Resource name as a String.
+          # +version+::  Endpoint version as an Integer.
+          #
+          def discover_remote( resource, version )
+            endpoint_uri_string = drb_service().find( resource, version )
+            return result_for( resource, version, endpoint_uri_string )
+          end
+
+        private
+
+          # Construct a Hoodoo::Services::Discovery::ForHTTP instance for
+          # the given parameters.
+          #
+          # +resource+::            Resource name as a String.
+          # +version+::             Endpoint version as an Integer.
+          # +endpoint_uri_string+:: Endpoint location as a URI expressed
+          #                         as a String; may be +nil+.
+          #
+          # Returns the new instance, or +nil+ if the endpoint URI String
+          # was itself +nil+.
+          #
+          def result_for( resource, version, endpoint_uri_string )
+            if endpoint_uri_string.nil?
+              return nil
+            else
+              return Hoodoo::Services::Discovery::ForHTTP.new(
+                resource:     resource,
+                version:      version,
+                endpoint_uri: URI.parse( endpoint_uri_string )
+              )
+            end
+          end
+
+          # Start the DRb service on the port configured for this instance
+          # via its constructor and return a DRbObject instance to use for
+          # talking to it.
+          #
+          # Raises an exception if the DRb service cannot be started.
+          #
+          def drb_service
+
+            # Attempt to contact the DRb server daemon. If it can't be
+            # contacted, try to start it first, then connect.
+
+            if @drb_uri.nil? || @drb_uri.empty?
+              drb_uri = Hoodoo::Services::Discovery::ByDRb::DRbServer.uri( @drb_port )
+              start_on_localhost_if_not_already_running = true
+            else
+              drb_uri = @drb_uri
+              start_on_localhost_if_not_already_running = false
+            end
+
+            begin
+              drb_service = DRbObject.new_with_uri( drb_uri )
+              drb_service.ping()
+
+            rescue DRb::DRbConnError
+              if start_on_localhost_if_not_already_running
+                script_path = File.join( File.dirname( __FILE__ ), 'drb_server_start.rb' )
+                command     = "bundle exec ruby '#{ script_path }'"
+                command    << " --port #{ @drb_port }" unless @drb_port.nil? || @drb_port.empty?
+
+                Process.detach( spawn( command ) )
+
+                begin
+                  Timeout::timeout( 5 ) do
+                    loop do
+                      begin
+                        drb_service = DRbObject.new_with_uri( drb_uri )
+                        drb_service.ping()
+                        break
+                      rescue DRb::DRbConnError
+                        sleep 0.1
+                      end
+                    end
+                  end
+
+                rescue Timeout::Error
+                  raise "Hoodoo::Services::Discovery::ByDRb timed out while waiting for DRb service registry to start on local port #{ @drb_port }"
+
+                end
+
+              else
+                raise "Hoodoo::Services::Discovery::ByDRb could not contact a DRb service registry at #{ @drb_uri }"
+
+              end
+            end
+
+            return drb_service
+          end
+
+      end
+    end
+  end
+end

data/lib/hoodoo/services/discovery/discoverers/by_drb/drb_server.rb

@@ -0,0 +1,166 @@
+########################################################################
+# File::    drb_server.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: For local development with no wider registration service
+#           like MemCache running, the first service that gets started
+#           will bring up a DRb server to record the location of other
+#           services as they start up. The subsequent services connect
+#           to the existing DRb server started by the first.
+#
+#           This class is almost a private implementation detail of
+#           Hoodoo::Services::Middleware and is namespaced inside it.
+#           File "service_middleware.rb" must be "require"'d first.
+# ----------------------------------------------------------------------
+#           11-Nov-2014 (ADH): Split out from service_middleware.rb.
+#           02-Mar-2015 (ADH): Moved into Discovery namespace.
+########################################################################
+
+require 'hoodoo'
+
+require 'drb/drb'
+require 'drb/acl'
+
+module Hoodoo; module Services; class Discovery # Just used as a namespace here
+  class ByDRb < Hoodoo::Services::Discovery # Also just used as a namespace here
+
+    # A registry of service endpoints, implenented as a DRB server class. An
+    # internal implementation detail of Hoodoo::Services::Middleware, in most
+    # respects.
+    #
+    class DRbServer
+
+      # URI for DRb server used during local machine development as a registry
+      # of service endpoints. Whichever service starts first runs the server
+      # which others connect to if subsequently started.
+      #
+      # +port+:: Optional integer port number for DRb service. If specified,
+      #          this is used; else the +HOODOO_DISCOVERY_BY_DRB_PORT_OVERRIDE+
+      #          environment variable is used; else a default of 8787 is
+      #          chosen. Passing +nil+ explicitly also leads to the use of
+      #          the environment variable or default value.
+      #
+      def self.uri( port = nil )
+
+        port ||= ENV[ 'HOODOO_DISCOVERY_BY_DRB_PORT_OVERRIDE' ] || 8787
+
+        # Use IP address, rather than 'localhost' here, to ensure that "address
+        # in use" errors are raised immediately if a second server startup
+        # attempt is made:
+        #
+        #   https://bugs.ruby-lang.org/issues/3052
+        #
+        "druby://127.0.0.1:#{ port }"
+
+      end
+
+      # Start the DRb server. Does not return (joins the DRb thread). If the
+      # server is already running, expect an "address in use" connection
+      # exception from DRb.
+      #
+      # $SAFE will be set to 1 (unless it is already higher) in this thread.
+      #
+      # +port+:: Passed to ::uri method.
+      #
+      def self.start( port = nil )
+
+        uri = self.uri( port )
+
+        # For security, "disable eval() and friends":
+        #
+        # http://www.ruby-doc.org/stdlib-2.2.3/libdoc/drb/rdoc/DRb.html
+        # https://ruby-hacking-guide.github.io/security.html
+        # http://blog.recurity-labs.com/archives/2011/05/12/druby_for_penetration_testers/
+
+        $SAFE = 1
+
+        # Have to allow a tained port string from "outside" just to be able
+        # to start the service on a given port; so untaint that deliberately.
+        #
+        # http://ruby-doc.com/docs/ProgrammingRuby/html/taint.html
+
+        uri.untaint()
+        $stop_queue = ::Queue.new
+
+        ::DRb.start_service( uri,
+                             FRONT_OBJECT,
+                             :tcp_acl => LOCAL_ACL )
+
+        # DRB.thread.exit() does not reliably work; sometimes, it just hangs
+        # up. I don't know why. On OS X and under Travis, sporadic failures
+        # to return from the "stop()" method would result. Instead, we use a
+        # relatively elaborate queue; sit here waiting for a message to be
+        # pushed onto it, then just let this method exit naturally, ignoring
+        # the value that appeared on the queue.
+        #
+        # The sleep makes it more reliable too, indicating some kind of nasty
+        # race condition on start-vs-wait-to-shutdown.
+
+        sleep( 1 )
+        $stop_queue.pop()
+      end
+
+      # Create an instance ready for use as a DRb "front object".
+      #
+      def initialize
+        @repository = {}
+      end
+
+      # Check to see if this DRb service is awake. Returns +true+.
+      #
+      def ping
+        return true
+      end
+
+      # Add an endpoint to the list. If the endpoint was already added,
+      # it will be overwritten with the new data.
+      #
+      # +resource+:: Resource as a String or Symbol, e.g. "Product"
+      # +version+::  Endpoint's implemented API version as an Integer, e.g. 1
+      # +uri+::      URI at which this service may be accessed, including the
+      #              endpoint path (e.g. "http://localhost:3002/v1/products"),
+      #              as a String.
+      #
+      def add( resource, version, uri )
+        @repository[ "#{ resource }/#{ version }" ] = uri
+      end
+
+      # Find an endpoint in the list. Returns URI at which the service may be
+      # accessed as a String, or 'nil' if not found.
+      #
+      # +resource+:: Resource as a String or Symbol, e.g. "Product"
+      # +version+::  Endpoint's implemented API version as an Integer, e.g. 1
+      #
+      def find( resource, version )
+        @repository[ "#{ resource }/#{ version }" ]
+      end
+
+      # Flush out the repository, clearing all stored service records. This is
+      # usually for test purposes only.
+      #
+      def flush
+        @repository = {}
+      end
+
+      # Shut down this DRb service.
+      #
+      def stop
+        $stop_queue.push( true )
+      end
+    end
+
+    # Singleton "Front object" for the DRB service used in local development.
+    #
+    FRONT_OBJECT = Hoodoo::Services::Discovery::ByDRb::DRbServer.new
+
+    # Only allow connections from 127.0.0.1.
+    #
+    LOCAL_ACL = ACL.new( %w[
+      deny all
+      allow ::1
+      allow fe80::1%lo0
+      allow 127.0.0.1
+    ] )
+
+  end
+end; end; end

data/lib/hoodoo/services/discovery/discoverers/by_drb/drb_server_start.rb

@@ -0,0 +1,37 @@
+########################################################################
+# File::    drb_server_start.rb
+# (C)::     Loyalty New Zealand 2014
+#
+# Purpose:: Run the DRB server. See service_registry_drb_server.rb.
+#           Usage:
+#
+#               bundle exec ruby drb_server_start.rb
+#
+#           There is usually no need to do this manually, as the
+#           middleware does it for you automatically.
+# ----------------------------------------------------------------------
+#           23-Dec-2014 (ADH): Created.
+########################################################################
+
+require 'ostruct'
+require 'optparse'
+
+require 'hoodoo'
+
+options = OpenStruct.new
+
+OptionParser.new do | opts |
+  opts.banner = 'Usage: drb_server_start.rb [options]'
+
+  opts.on( '-p', '--port PORT', 'Listening port' ) do | val |
+    options.port = val || ENV[ 'HOODOO_DISCOVERY_BY_DRB_PORT_OVERRIDE' ] || 8787
+  end
+
+  opts.on( '-h', '--help', 'Prints this help' ) do
+    puts opts
+    exit
+  end
+end.parse!
+
+Process.setsid()
+Hoodoo::Services::Discovery::ByDRb::DRbServer.start( options.port )

data/lib/hoodoo/services/discovery/discovery.rb

@@ -0,0 +1,186 @@
+########################################################################
+# File::    discovery.rb
+# (C)::     Loyalty New Zealand 2015
+#
+# Purpose:: Support resource endpoint discovery.
+# ----------------------------------------------------------------------
+#           03-Mar-2015 (ADH): Created.
+########################################################################
+
+module Hoodoo
+  module Services
+
+    # The service discovery mechanism is a way to find Resource
+    # implementations running inside service applications that may be
+    # available at HTTP URIs, over an AMQP queue or, potentially, any other
+    # system. Subclasses implement a particular distinct discovery approach.
+    # When implementations of services start up, they announce themselves
+    # (via Hoodoo::Services::Middleware) to the discovery engine. When other
+    # Resources (or Hoodoo::Client) want to find them, they query the same
+    # discovery engine to find out the original announcement information.
+    #
+    # Depending on how a discovery engine shares information about
+    # announced Resource endpoints, Resources might only be found if they are
+    # are on the same local machine; or the same remote host or queue; or
+    # they might perhaps be available even if scattered across multiple hosts
+    # and/or transport types.
+    #
+    # Implementations of service announcement and discovery code must be a
+    # subclass of this class, then optionally implement #configure_with and
+    # (almost certainly, but still optionally) #announce_remote; and must
+    # always implement #discover_remote.
+    #
+    class Discovery
+
+      public
+
+        # Create a new instance.
+        #
+        # +options+:: Passed to the subclass in use via #configure_with.
+        #             Subclasses define their options. Only instantiate
+        #             such subclasses, not this 'Base' class; see the
+        #             subclass documentation for option details.
+        #
+        def initialize( options = {} )
+          @known_local_resources = {}
+          configure_with( options )
+        end
+
+        # Indicate that a resource is available locally and broacast its
+        # location to whatever discovery service a subclass supports via
+        # #announce_remote.
+        #
+        # +resource+:: Resource name as a Symbol or String (e.g.
+        #              +:Purchase+).
+        #
+        # +version+::  Endpoint version as an Integer; optional; default
+        #              is 1.
+        #
+        # +options+::  Defined by whatever subclass is in use. See that
+        #              subclass's documentation for details.
+        #
+        # Returns the result of calling #announce_remote (in the subclass
+        # in use) with the same parameters. See the protected method
+        # definition in this base class for details.
+        #
+        def announce( resource, version = 1, options = {} )
+          resource = resource.to_sym
+          version  = version.to_i
+          result   = announce_remote( resource, version, options )
+
+          @known_local_resources[ resource ] ||= {}
+          @known_local_resources[ resource ][ version ] = result
+
+          return result
+        end
+
+        # Find a resource endpoint. This may be recorded locally or
+        # via whatever remote discovery mechanism a subclass implements.
+        #
+        # +resource+:: Resource name as a Symbol or String (e.g.
+        #              +:Purchase+).
+        #
+        # +version+::  Endpoint version as an Integer; optional; default
+        #              is 1.
+        #
+        # Returns the result of calling #discover_remote (in the subclass
+        # in use) with the same parameters. See the protected method
+        # definition in this base class for details.
+        #
+        # Use #is_local? if you need to know that an endpoint was
+        # announced through this same instance ("locally").
+        #
+        def discover( resource, version = 1 )
+          resource = resource.to_sym
+          version  = version.to_i
+
+          if ( is_local?( resource, version ) )
+            return @known_local_resources[ resource ][ version ]
+          else
+            return discover_remote( resource, version )
+          end
+        end
+
+        # Was a resource announced in this instance ("locally")? Returns
+        # +true+ if so, else +false+.
+        #
+        # This only returns +true+ if #annouce has been called for the
+        # given resource and version.
+        #
+        # +resource+:: Resource name as a Symbol or String (e.g.
+        #              +:Purchase+).
+        #
+        # +version+::  Endpoint version as an Integer; optional; default
+        #              is 1.
+        #
+        def is_local?( resource, version = 1 )
+          resource = resource.to_sym
+          version  = version.to_i
+
+          return @known_local_resources.has_key?( resource ) &&
+                 @known_local_resources[ resource ].has_key?( version )
+        end
+
+      protected
+
+        # Configure a new instance. Subclasses optionally implement this
+        # method to store configuration information relevant to that
+        # subclass. Subclasses must document their options.
+        #
+        # +options+:: See subclass documentation for option details.
+        #
+        def configure_with( options )
+          # Implementation is optional and up to subclasses to do.
+        end
+
+        # Announce a resource endpoint. Subclasses optionally implement
+        # this method to broadcast information to other instances of the
+        # same subclass by some subclass-implemented mechanism.
+        #
+        # Discovery instance users do not call this method directly.
+        # Call #announce instead.
+        #
+        # Subclasses must return one of the Discovery "For" class instances,
+        # e.g. a Hoodoo::Services::Discovery::ForHTTP or
+        # Hoodoo::Services::Discovery::ForAMQP instance. This encapsulates
+        # the HTTP details required to contact the endpoint, or AMQP (queue)
+        # details required to contact the endpoint, respectively.
+        #
+        # Subclasses must return +nil+ if it has a problem announcing and
+        # cannot provide information for the given resource / version.
+        #
+        # +resource+:: Resource name as a String.
+        # +version+::  Endpoint version as an Integer.
+        # +options+::  See subclass documentation for option details.
+        #
+        def announce_remote( resource, version, options = {} )
+          # Implementation is optional and up to subclasses to do.
+          nil
+        end
+
+        # Discover the location of a resource endpoint. Subclasses _must_
+        # implement this method to retrieve information about the location
+        # of resource endpoints by some subclass-implemented mechanism.
+        #
+        # Discovery instance users do not call this method directly.
+        # Call #discover instead.
+        #
+        # Subclasses must return one of the Discovery "For" class instances,
+        # e.g. a Hoodoo::Services::Discovery::ForHTTP or
+        # Hoodoo::Services::Discovery::ForAMQP instance. This encapsulates
+        # the HTTP details required to contact the endpoint, or AMQP (queue)
+        # details required to contact the endpoint, respectively.
+        #
+        # If the requested endpoint is not found, subclasses must return
+        # +nil+.
+        #
+        # +resource+:: Resource name as a String.
+        # +version+::  Endpoint version as an Integer.
+        #
+        def discover_remote( resource, version )
+          raise "Hoodoo::Services::Discovery::Base subclass does not implement remote discovery required for resource '#{ resource }' / version '#{ version }'"
+        end
+
+    end
+  end
+end