alchemy-flux 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9e0b6e71da840d69a7c637a0cb924df1c63fcf53
+  data.tar.gz: cb5572b0ce8972212d500e7f53e3234f9f7659f0
+SHA512:
+  metadata.gz: eb865f0288c6b2c3de1f4947b4440b35727bc48d1bb7d8f3029cbcebf5b70784384f487924fd66ff7a7e25723528ef59c88d77c8078c89c5f05cf16dd8c6930e
+  data.tar.gz: 725f8ee5b7ce199d5ea70fdc913c60aff118f085f7d76085a175e16f42161c7d126eceb10b550ac1f08245bbf83f90bc0868cb81c2dd47972eb5ab9287112907

data/lib/alchemy-flux.rb

@@ -0,0 +1,409 @@
+require 'time'
+require 'amqp'
+require "uuidtools"
+require 'msgpack'
+
+require 'alchemy-flux/flux_rack_handler.rb'
+
+# Alchemy Flux module
+module AlchemyFlux
+
+  # Error created when a Service message times out
+  class TimeoutError < StandardError; end
+
+  # Error created when a Message is unable to be delivered to a service
+  class MessageNotDeliveredError < StandardError; end
+
+  # Error used by a service when they wish the calling message to be NACKed *dangerous*
+  class NAckError < StandardError; end
+
+  # An Alchemy Flux Service
+  class Service
+
+    # The current state of the Service, either *stopped* or *started*
+    attr_reader :state
+
+    # The outgoing message transactions
+    attr_reader :transactions
+
+    # The incoming number of messages being processed
+    attr_reader :processing_messages
+
+    # Generate a UUID string
+    def self.generateUUID
+      UUIDTools::UUID.random_create.to_i.to_s(16).ljust(32,'0')
+    end
+
+    # Create a AlchemyFlux service instance
+    #
+    # +name+ the name of the service being created
+    # +options+
+    #
+    def initialize(name, options = {}, &block)
+      @name = name
+      @options = {
+          ampq_uri: 'amqp://localhost',
+          prefetch: 20,
+          timeout: 1000,
+          threadpool_size: 500,
+          resource_paths: []
+      }.merge(options)
+
+      @service_fn = block || Proc.new { |message| "" }
+
+      @uuid = "#{@name}.#{AlchemyFlux::Service.generateUUID()}"
+      @transactions = {}
+      @processing_messages = 0
+
+      @response_queue_name = @uuid
+      @service_queue_name = @name
+      @state = :stopped
+    end
+
+    # overriding inspect
+    def inspect
+      to_s
+    end
+
+    # overriding to_s
+    def to_s
+      "AlchemyFlux::Service(#{@name.inspect}, #{@options.inspect})"
+    end
+
+    # LIFE CYCLE
+
+
+    # Start the EventMachine and AMQP connections for all Services
+    #
+    # The application has two or more threads
+    # 1. The Controller Thread (e.g. the rspec thread)
+    # 2. The EM Thread
+    # 3. The EM defer Threads
+    #
+    # When we start a Service we do it in a Thread so that it will not block the calling Thread
+    #
+    # When the FIRST Service is started EM.run initialises in that Thread
+    # When the second Service is initialises the block is executed in the new thread,
+    # but all the callbacks will be executed in the EM thread
+    #
+    def self.start(ampq_uri = 'amqp://localhost', threadpool_size=500)
+      return if EM.reactor_running?
+      start_blocker = Queue.new
+      Thread.new do
+        Thread.current["name"] = "EM Thread" if EM.reactor_thread?
+        Thread.current.abort_on_exception = true
+        EM.threadpool_size = threadpool_size
+        AMQP.start(ampq_uri) do |connection|
+          @@connection = connection
+          @@connection.on_error do |conn, connection_close|
+            message = "Channel exception: [#{connection_close.reply_code}] #{connection_close.reply_text}"
+            puts message
+            raise message
+          end
+          start_blocker << :unblock
+        end
+      end
+      start_blocker.pop
+    end
+
+    # Stop EventMachine and the
+    def self.stop
+      return if !EM.reactor_running?
+      stop_blocker = Queue.new
+
+      #last tick
+      AMQP.stop do
+        EM.stop_event_loop
+        stop_blocker << :unblock
+      end
+      stop_blocker.pop
+      sleep(0.05) # to ensure it finished
+    end
+
+    # start the service
+    def start
+      return if @state != :stopped
+
+      Service.start(@options[:ampq_uri], @options[:threadpool_size])
+      EM.run do
+
+        @channel = AMQP::Channel.new(@@connection)
+
+        @channel.on_error do |ch, channel_close|
+          message = "Channel exception: [#{channel_close.reply_code}] #{channel_close.reply_text}"
+          puts message
+          raise message
+        end
+
+        @channel.prefetch(@options[:prefetch])
+        @channel.auto_recovery = true
+
+        @service_queue = @channel.queue( @service_queue_name, {:durable => true})
+        @service_queue.subscribe({:ack => true}) do |metadata, payload|
+          payload = MessagePack.unpack(payload)
+          process_service_queue_message(metadata, payload)
+        end
+
+        response_queue = @channel.queue(@response_queue_name, {:exclusive => true, :auto_delete => true})
+        response_queue.subscribe({}) do |metadata, payload|
+          payload = MessagePack.unpack(payload)
+          process_response_queue_message(metadata, payload)
+        end
+
+        @channel.default_exchange.on_return do |basic_return, frame, payload|
+          payload = MessagePack.unpack(payload)
+          process_returned_message(basic_return, frame.properties, payload)
+        end
+
+        # RESOURCES HANDLE
+        @resources_exchange = @channel.topic("resources.exchange", {:durable => true})
+        @resources_exchange.on_return do |basic_return, frame, payload|
+          payload = MessagePack.unpack(payload)
+          process_returned_message(basic_return, frame.properties, payload)
+        end
+
+        bound_resources = 0
+        for resource_path in @options[:resource_paths]
+          binding_key = "#{path_to_routing_key(resource_path)}.#"
+          @service_queue.bind(@resources_exchange, :key => binding_key) {
+            bound_resources += 1
+          }
+        end
+        begin
+          # simple loop to wait for the resources to be bound
+          sleep(0.01)
+        end until bound_resources == @options[:resource_paths].length
+
+        @state = :started
+      end
+    end
+
+    # Stop the Service
+    #
+    # This method:
+    # * Stops receiving new messages
+    # * waits for processing incoming and outgoing messages to be completed
+    # * close the channel
+    def stop
+      return if @state != :started
+      # stop receiving new incoming messages
+      @service_queue.unsubscribe
+      # only stop the service if all incoming and outgoing messages are complete
+      decisecond_timeout = @options[:timeout]/100
+      waited_deciseconds = 0 # guarantee that this loop will stop
+      while (@transactions.length > 0 || @processing_messages > 0) && waited_deciseconds < decisecond_timeout
+        sleep(0.1) # wait a decisecond to check the incoming and outgoing messages again
+        waited_deciseconds += 1
+      end
+
+      @channel.close
+      @state = :stopped
+    end
+
+    # END OF LIFE CYCLE
+
+
+
+    private
+    # RECIEVING MESSAGES
+
+    # process messages on the service queue
+    def process_service_queue_message(metadata, payload)
+
+      service_to_reply_to = metadata.reply_to
+      message_replying_to = metadata.message_id
+      this_message_id = AlchemyFlux::Service.generateUUID()
+      delivery_tag = metadata.delivery_tag
+      interaction_id = payload['headers']['x-interaction-id']
+
+      operation = proc {
+        @processing_messages += 1
+        begin
+          response = @service_fn.call(payload)
+          {
+            'status_code' => response['status_code'] || 200,
+            'headers' => response['headers']         || { 'x-interaction-id' => interaction_id},
+            'body' => response['body']               || ""
+          }
+        rescue AlchemyFlux::NAckError => e
+          AlchemyFlux::NAckError
+        rescue Exception => e
+          puts "Service Fn Error " + e.inspect
+
+          # Returning Hoodoo formatted Error code (just in case service doesnt handle the error, it should!)
+          {
+            'status_code' => 500,
+            'headers' => {'Content-Type' => 'application/json; charset=utf-8'},
+            'body' =>   {
+              'kind' =>           "Errors",
+              'id' =>             AlchemyFlux::Service.generateUUID(),
+              'created_at' =>     Time.now.utc.iso8601,
+              'interaction_id' => interaction_id,
+              'errors' => [{'code' => 'platform.fault', 'message' => 'An unexpected error occurred'}]
+            }
+          }
+        end
+      }
+
+      callback = proc { |result|
+
+        if result == AlchemyFlux::NAckError
+          @service_queue.reject(delivery_tag)
+        else
+          options = {
+            :message_id     => this_message_id,
+            :correlation_id => message_replying_to,
+            :type           =>    'http_response'
+          }
+          send_message(@channel.default_exchange, service_to_reply_to, result, options)
+          @processing_messages -= 1
+          @service_queue.acknowledge(delivery_tag)
+        end
+      }
+
+      EventMachine.defer(operation, callback)
+    end
+
+    # process a response message
+    #
+    # If a message is put on this services response queue
+    # its response will be pushed onto the blocking queue
+    def process_response_queue_message(metadata, payload)
+      response_queue = @transactions.delete metadata.correlation_id
+      response_queue << payload if response_queue
+    end
+
+    # process a returned message
+    #
+    # If a message is sent to a queue that cannot be found,
+    # rabbitmq returns that message to this method
+    def process_returned_message(basic_return, metadata, payload)
+      response_queue = @transactions.delete metadata[:message_id]
+      response_queue << MessageNotDeliveredError if response_queue
+    end
+
+    # END OF RECIEVING MESSAGES
+
+    public
+
+    # SENDING MESSAGES
+
+    # send a message to an exchange with routing key
+    #
+    # *exchange*:: A AMQP exchange
+    # *routing_key*:: The routing key to use
+    # *message*:: The message to be sent
+    # *options*:: The message options
+    def send_message(exchange, routing_key, message, options)
+      message_options = options.merge({:routing_key => routing_key})
+      message = MessagePack.pack(message)
+      EventMachine.next_tick do
+        exchange.publish message, message_options
+      end
+    end
+
+    # send a message to a service
+    #
+    # *service_name*:: the name of the service
+    # *message*:: the message to be sent
+    #
+    # This method can optionally take a block which will be executed asynchronously and yielded the response
+    def send_message_to_service(service_name, message)
+      if block_given?
+        EventMachine.defer do
+          yield send_message_to_service(service_name, message)
+        end
+      else
+        send_HTTP_request_message(@channel.default_exchange, service_name, message)
+      end
+    end
+
+    # send a message to a resource
+    #
+    # *message*:: the message to be sent to the *path* in the message
+    #
+    # This method can optionally take a block which will be executed asynchronously and yielded the response
+    def send_message_to_resource(message)
+      routing_key = path_to_routing_key(message['path'])
+      if block_given?
+        EventMachine.defer do
+          yield send_message_to_resource(message)
+        end
+      else
+        send_HTTP_request_message(@resources_exchange, routing_key, message)
+      end
+    end
+
+
+    private
+
+    # Takes a path and converts it into a routing key
+    #
+    # *path*:: path string
+    #
+    # For example, path '/test/path' will convert to routing key 'test.path'
+    def path_to_routing_key(path)
+      new_path = ""
+      path.split('').each_with_index do |c,i|
+        if c == '/' and i != 0 and i != path.length-1
+          new_path += '.'
+        elsif c != '/'
+          new_path += c
+        end
+      end
+      new_path
+    end
+
+    # send a HTTP message to an exchange with routing key
+    #
+    # *exchange*:: A AMQP exchange
+    # *routing_key*:: The routing key to use
+    # *message*:: The message to be sent
+    def send_HTTP_request_message(exchange, routing_key, message)
+
+      http_message = {
+        'session_id' =>  message['session_id'],
+        'scheme' =>      message['protocol']    || 'http',
+        'host' =>        message['hostname']    || 'localhost',
+        'port' =>        message['port']        || 8080,
+        'path' =>        message['path']        || "/",
+        'query' =>       message['query']       || {},
+        'verb' =>        message['verb']        || "GET",
+        'headers' =>     message['headers']     || {},
+        'body' =>        message['body']        || ""
+      }
+
+      if !http_message['headers']['x-interaction-id']
+        http_message['headers']['x-interaction-id'] = AlchemyFlux::Service.generateUUID()
+      end
+
+      message_id = AlchemyFlux::Service.generateUUID()
+
+      http_message_options = {
+        message_id:          message_id,
+        type:               'http_request',
+        reply_to:            @response_queue_name,
+        content_encoding:    '8bit',
+        content_type:        'application/octet-stream',
+        expiration:           @options[:timeout],
+        mandatory:            true
+      }
+
+      response_queue = Queue.new
+      @transactions[message_id] = response_queue
+
+      send_message(exchange, routing_key, http_message, http_message_options)
+
+      EventMachine.add_timer(@options[:timeout]/1000.0) do
+        response_queue = @transactions.delete message_id
+        response_queue << TimeoutError if response_queue
+      end
+
+      response_queue.pop
+    end
+
+    # END OF SENDING MESSAGES
+
+  end
+
+end

data/lib/alchemy-flux/flux_rack_handler.rb

@@ -0,0 +1,159 @@
+# This file creates a Service that Talks Rack
+require 'rack'
+require 'alchemy-flux'
+
+# The Rack namespace
+module Rack
+  # The Rack Handlers namespace
+  module Handler
+    # Alchemy Rack handler
+    class AlchemyFlux
+
+
+
+      # Start the app server with the supplied Rack application and options
+      #
+      # +app+ [Rack Application] The Application to run.
+      # +options+ [Hash] The options to start the server with.
+      def self.run(app, options={})
+        start(app)
+
+        puts "Started #{@@service.inspect}"
+
+        Signal.trap("INT")  do
+          puts "Stopping #{@@service.inspect}"
+          stop
+        end
+
+        Signal.trap("TERM") do
+          puts "Stopping #{@@service.inspect}"
+          stop
+        end
+
+        EM.reactor_thread.join
+      end
+
+      # start the service for rack
+      def self.start(app)
+        service_name = ENV['SERVICE_NAME']
+        raise RuntimeError.new("Require SERVICE_NAME environment variable") if !service_name
+
+        options = {
+          ampq_uri: ENV['AMQ_URI'] || 'amqp://localhost',
+          prefetch: ENV['PREFETCH'] || 20,
+          timeout: ENV['TIMEOUT'] || 30000,
+          threadpool_size: ENV['THREADPOOL_SIZE'] || 500,
+          resource_paths: (ENV['RESOURCE_PATHS'] || '').split(',')
+        }
+
+        if options[:prefetch] > options[:threadpool_size]
+          puts "WARNING: 'prefect' is greater than the available threads which may cause performance blocking problems"
+        end
+
+        @@service = ::AlchemyFlux::Service.new(service_name, options) do |message|
+          rack_env = create_rack_env(message)
+
+          # add Alchemy Service so the app may call other services
+          rack_env['alchemy.service'] = @@service
+
+          status, headers, body = app.call(rack_env)
+
+          # process the body into a single response string
+          body.close if body.respond_to?(:close)
+          response = ""
+          body.each { |part| response << part }
+
+          {
+            'status_code' => status,
+            'headers' => headers,
+            'body' => response
+          }
+        end
+
+
+        @@service.start
+      end
+
+      # stops the app service
+      def self.stop
+        @@service.stop
+        EM.stop
+      end
+
+      # create the environment hash to be sent to the app
+      def self.create_rack_env(message)
+
+        stream = StringIO.new(message['body'])
+        stream.set_encoding(Encoding::ASCII_8BIT)
+
+
+        # Full description of rack env http://www.rubydoc.info/github/rack/rack/master/file/SPEC
+        rack_env = {}
+
+        # CGI-like (adopted from PEP333) variables
+
+        # The HTTP request method, such as “GET” or “POST”
+        rack_env['REQUEST_METHOD'] = message['verb'].to_s.upcase
+
+        # This is an empty string to correspond with the “root” of the server.
+        rack_env['SCRIPT_NAME'] = ''
+
+        # The remainder of the request URL's “path”, designating the virtual “location” of the request's target within the application.
+        rack_env['PATH_INFO'] = message['path']
+
+        # The portion of the request URL that follows the ?, if any
+        rack_env['QUERY_STRING'] = Rack::Utils.build_query(message['query'])
+
+        # Used to complete the URL
+        rack_env['SERVER_NAME'] = message['host']
+        rack_env['SERVER_PORT'] = message['port'].to_s
+
+
+        # Headers are added to the rack env as described by RFC3875 https://www.ietf.org/rfc/rfc3875
+        if message['headers'].is_a? Hash
+          message['headers'].each do |name, value|
+            name = "HTTP_" + name.to_s.upcase.gsub(/[^A-Z0-9]/,'_')
+            rack_env[name] = value.to_s
+          end
+        end
+
+        # The environment must not contain the keys HTTP_CONTENT_TYPE or HTTP_CONTENT_LENGTH (use the versions without HTTP_)
+        rack_env['CONTENT_TYPE'] = rack_env['HTTP_CONTENT_TYPE'] || 'application/octet-stream'
+        rack_env['CONTENT_LENGTH'] = rack_env['HTTP_CONTENT_LENGTH'] || stream.length.to_s
+        rack_env.delete('HTTP_CONTENT_TYPE')
+        rack_env.delete('HTTP_CONTENT_LENGTH')
+
+
+        # Rack-specific variables
+
+        # The Array representing this version of Rack See Rack::VERSION
+        rack_env['rack.version'] = Rack::VERSION
+
+        # http or https, depending on the request URL.
+        rack_env['rack.url_scheme'] = message['scheme']
+
+        # the input stream.
+        rack_env['rack.input'] = stream
+
+        # the error stream.
+        rack_env['rack.errors'] = STDERR
+
+        # true if the application object may be simultaneously invoked by another thread in the same process, false otherwise.
+        rack_env['rack.multithread'] = true
+
+        # true if an equivalent application object may be simultaneously invoked by another process, false otherwise.
+        rack_env['rack.multiprocess'] = false
+
+        # true if the server expects (but does not guarantee!) that the application will only be invoked this one time during the life of its containing process.
+        rack_env['rack.run_once'] = false
+
+        # present and true if the server supports connection hijacking.
+        rack_env['rack.hijack?'] = false
+
+        rack_env
+      end
+    end
+
+    register :alchemy, Rack::Handler::AlchemyFlux
+  end
+end