asynk 0.0.1

data/lib/asynk/cli.rb

@@ -0,0 +1,146 @@
+$stdout.sync = true
+
+require 'optparse'
+require 'fileutils'
+require 'asynk'
+module Asynk
+  class CLI
+    include Singleton
+    attr_accessor :environment
+
+    def run(args=ARGV)
+      Asynk.booted_inside = true
+      setup_options(args)
+      initialize_logger
+      daemonize
+      write_pid
+      load_celluloid
+      boot_system
+      Asynk.server.run
+    end
+
+    private
+
+      def load_celluloid
+        raise "Celluloid cannot be required until here, or it will break Sidekiq's daemonization" if defined?(::Celluloid) && options[:daemon]
+
+        # Celluloid can't be loaded until after we've daemonized
+        # because it spins up threads and creates locks which get
+        # into a very bad state if forked.
+        require 'celluloid'
+        require 'celluloid/io'
+        Celluloid.logger = (options[:verbose] ? Asynk.logger : nil)
+      end
+
+      def daemonize
+        return unless options[:daemon]
+
+        raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
+        files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+          files_to_reopen << file unless file.closed?
+        end
+
+        ::Process.daemon(true, true)
+
+        files_to_reopen.each do |file|
+          begin
+            file.reopen file.path, "a+"
+            file.sync = true
+          rescue ::Exception
+          end
+        end
+
+        [$stdout, $stderr].each do |io|
+          File.open(options[:logfile], 'ab') do |f|
+            io.reopen(f)
+          end
+          io.sync = true
+        end
+        $stdin.reopen('/dev/null')
+
+        initialize_logger
+      end
+
+      def write_pid
+        if path = options[:pidfile]
+          pidfile = File.expand_path(path)
+          File.open(pidfile, 'w') do |f|
+            f.puts ::Process.pid
+          end
+        end
+      end
+
+      def options
+        Asynk.options
+      end
+
+      def initialize_logger
+        Asynk::Logging.initialize_logger(options[:logfile]) if options[:logfile]
+        Asynk.logger.level = ::Logger::DEBUG if options[:verbose]
+      end
+
+      def setup_options(args)
+        opts = parse_options(args)
+        set_environment opts[:environment]
+        options.merge!(opts)
+      end
+
+      def boot_system
+        Asynk.logger.info 'Booting Asynk App'
+        ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
+
+        raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
+
+        if File.directory?(options[:require])
+          require File.expand_path("#{options[:require]}/config/application.rb")
+          require File.expand_path("#{options[:require]}/config/environment.rb")
+          ::Rails.application.eager_load!
+        else
+          require options[:require]
+        end
+      end
+
+      def set_environment(cli_env)
+        @environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+      end
+
+      def parse_options(argv)
+        opts = {}
+
+        @parser = OptionParser.new do |o|
+          o.on '-d', '--daemon', "Daemonize process" do |arg|
+            opts[:daemon] = arg
+          end
+
+          o.on '-e', '--environment ENV', "Application environment" do |arg|
+            opts[:environment] = arg
+          end
+
+          o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
+            opts[:require] = arg
+          end
+
+          o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
+            opts[:logfile] = arg
+          end
+
+          o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
+            opts[:pidfile] = arg
+          end
+
+          o.on '-V', '--version', "Print version and exit" do |arg|
+            puts "Asynk #{Asynk::VERSION}"
+            exit(0)
+          end
+
+          o.on "-v", "--verbose", "Print more verbose output" do |arg|
+            opts[:verbose] = arg
+          end
+        end
+
+        @parser.parse!(argv)
+        opts
+      end
+  end
+end

data/lib/asynk/config.rb

@@ -0,0 +1,33 @@
+module Asynk
+  class Config
+    include Singleton
+    def initialize
+      @params = {
+                      mq_exchange: 'asynk_exchange_topic',
+        sync_publish_wait_timeout: 10,
+     default_consumer_concurrency: 1,
+                     default_sync: false,
+                        daemonize: false,
+                          logfile: 'log/asynk.log',
+                          pidifle: 'tmp/pids/asynk.pid',
+                          mq_host: 'localhost',
+                          mq_port: 5672,
+                         mq_vhost: '/',
+                      mq_username: 'guest',
+                      mq_password: 'guest',
+          publisher_execution_time: true,
+       respond_back_execution_time: true,
+                ignored_consumers: []
+      }
+    end
+
+    def [](key)
+      @params[key]
+    end
+
+    def []=(key, value)
+      @params[key] = value
+    end
+
+  end
+end

data/lib/asynk/consumer.rb

@@ -0,0 +1,98 @@
+module Asynk
+  module Consumer
+
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.include ActiveSupport::Rescuable
+      Asynk.register_consumer(base)
+    end
+
+    def initialize(channel, delivery_info, &block)
+      @channel = channel
+      @delivery_info = delivery_info
+      @callback_block = block
+    end
+
+    def ack!
+      @channel.ack(@delivery_info.delivery_tag)
+    end
+
+    def reject!
+      @channel.reject(@delivery_info.delivery_tag)
+    end
+
+    def requeue!
+      @channel.reject(@delivery_info.delivery_tag, true)
+    end
+
+    def logger
+      Asynk.logger
+    end
+
+    def respond(result)
+      @callback_block.call(result)
+    end
+
+    def invoke_processing(message)
+      method_for_exec = (self.class.route_ending_as_action? && message.routing_key) ?
+                          self.class.action_name_from_routing_key(message.routing_key) : :process
+
+      begin
+        public_send(method_for_exec, message)
+      rescue Exception => ex
+        raise(ex) unless rescue_with_handler(ex)
+      end
+    end
+
+    module ClassMethods
+      attr_reader :routing_keys, :subscribe_arguments, :queue_options
+
+      def set_consume(*routing_keys)
+        @routing_keys = routing_keys
+      end
+
+      def set_route_ending_as_action(value)
+        @route_ending_as_action = value
+      end
+
+      def route_ending_as_action?
+        @route_ending_as_action || false
+      end
+
+      def set_queue_name(options = {})
+        @queue_name = name
+      end
+
+      def set_queue_options(options = {})
+        @queue_options = options
+      end
+
+      def queue_name
+        return @queue_name unless @queue_name.nil?
+        app_name = Rails.application.class.parent_name.dup.underscore if defined?(Rails)
+        queue_name = ActiveSupport::Inflector.underscore(self.name.gsub(/::/, '.'))
+        queue_name = [app_name, queue_name].join('.') if app_name
+        queue_name
+      end
+
+      def set_subscribe_arguments(arguments = {})
+        @subscribe_arguments = arguments
+      end
+
+      def set_concurrency(size)
+        @concurrency = size
+      end
+
+      def concurrency
+        @concurrency || Asynk.config[:default_consumer_concurrency]
+      end
+
+      def action_name_from_routing_key(routing_key)
+        splitted = routing_key.split('.')
+        raise 'There now action in routing_key' if splitted.empty? || splitted.count < 2
+        splitted.last.to_sym
+      end
+
+    end
+  end
+end

data/lib/asynk/logging.rb

@@ -0,0 +1,97 @@
+require 'time'
+require 'logger'
+
+module Asynk
+  class Logging
+
+    class Pretty < Logger::Formatter
+      SPACE = " "
+
+      # Provide a call() method that returns the formatted message.
+      def call(severity, time, program_name, message)
+        "#{time.utc.iso8601(3)} #{severity}: #{message}\n"
+      end
+
+      def context
+        c = Thread.current[:sidekiq_context]
+        " #{c.join(SPACE)}" if c && c.any?
+      end
+    end
+
+    class WithoutTimestamp < Pretty
+      def call(severity, time, program_name, message)
+         #{severity}: #{message}\n"
+      end
+    end
+
+    def self.initialize_logger(log_target = STDOUT)
+      oldlogger = defined?(@logger) ? @logger : nil
+      @logger = Logger.new(log_target)
+      @logger.level = Logger::INFO
+      @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
+      oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
+      @logger
+    end
+
+    def self.logger
+      defined?(@logger) ? @logger : initialize_logger
+    end
+
+    def self.logger=(log)
+      @logger = (log ? log : Logger.new('/dev/null'))
+    end
+
+    # This reopens ALL logfiles in the process that have been rotated
+    # using logrotate(8) (without copytruncate) or similar tools.
+    # A +File+ object is considered for reopening if it is:
+    #   1) opened with the O_APPEND and O_WRONLY flags
+    #   2) the current open file handle does not match its original open path
+    #   3) unbuffered (as far as userspace buffering goes, not O_SYNC)
+    # Returns the number of files reopened
+    def self.reopen_logs
+      to_reopen = []
+      append_flags = File::WRONLY | File::APPEND
+
+      ObjectSpace.each_object(File) do |fp|
+        begin
+          if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
+            to_reopen << fp
+          end
+        rescue IOError, Errno::EBADF
+        end
+      end
+
+      nr = 0
+      to_reopen.each do |fp|
+        orig_st = begin
+          fp.stat
+        rescue IOError, Errno::EBADF
+          next
+        end
+
+        begin
+          b = File.stat(fp.path)
+          next if orig_st.ino == b.ino && orig_st.dev == b.dev
+        rescue Errno::ENOENT
+        end
+
+        begin
+          File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+          fp.sync = true
+          nr += 1
+        rescue IOError, Errno::EBADF
+          # not much we can do...
+        end
+      end
+      nr
+    rescue RuntimeError => ex
+      # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
+      puts "Unable to reopen logs: #{ex.message}"
+    end
+
+    def logger
+      Sidekiq::Logging.logger
+    end
+
+  end
+end

data/lib/asynk/message.rb

@@ -0,0 +1,30 @@
+require 'active_support/core_ext/hash/indifferent_access'
+require 'forwardable'
+
+module Asynk
+  class Message
+    extend Forwardable
+
+    attr_reader :delivery_info, :properties, :payload, :body
+
+    def initialize(delivery_info, properties, payload)
+      @delivery_info = delivery_info
+      @properties    = properties
+      @payload       = payload
+      @body          = JSON.parse(payload).with_indifferent_access
+    end
+
+    def_delegator :@body, :[]
+    def_delegators :@properties, :message_id, :timestamp
+    def_delegators :@delivery_info, :routing_key, :exchange
+
+
+    def to_s
+      attrs = { :@body => body.to_s, message_id: message_id,
+                timestamp: timestamp, routing_key: routing_key }
+      "#<Message #{attrs.map { |k,v| "#{k}=#{v.inspect}" }.join(', ')}>"
+    end
+
+    alias_method :inspect, :to_s
+  end
+end

data/lib/asynk/publisher.rb

@@ -0,0 +1,28 @@
+module Asynk
+  class Publisher
+    class PublisherError < RuntimeError; end
+
+    class << self
+      def publish(routing_key, params = {})
+        global_start_time = Asynk::Benchmark.start if Asynk.config[:publisher_execution_time]
+        message_id = params.delete(:message_id) || generate_message_id
+
+        Asynk.broker.pool.with do |channel, exchange, reply_queue|
+          exchange.publish(params.to_json, message_id: message_id, routing_key: routing_key)
+        end
+
+        if Asynk.config[:publisher_execution_time]
+          Asynk.logger.info "Sending async message #{routing_key}:#{message_id} with params: #{params}. Completed In: #{Asynk::Benchmark.end(global_start_time)} ms."
+        end
+      end
+
+      def sync_publish(routing_key, params = {})
+        Asynk::SyncPublisher.new(routing_key, params).send
+      end
+
+      def generate_message_id(legnth = 8)
+        SecureRandom.hex(legnth)
+      end
+    end
+  end
+end

data/lib/asynk/response.rb

@@ -0,0 +1,57 @@
+require 'json'
+module Asynk
+  class Response
+    attr_reader :status, :body, :error_message
+
+    def initialize(status: , body: nil, error_message: nil)
+      @status, @body, @error_message = status, body, error_message
+    end
+
+    def success?; @status.to_s == 'ok'; end
+    def fail?; !success?; end
+
+    def to_h(options = {})
+      { status: @status, body: @body, error_message: @error_message }
+    end
+
+    def status
+      ActiveSupport::StringInquirer.new(@status)
+    end
+
+    def errors; @error_message; end
+
+    def [](key)
+      @body[key]
+    end
+
+    def to_s
+      to_h
+    end
+
+    def to_json
+      to_h.to_json
+    end
+
+    alias_method :to_s, :to_h
+    alias_method :as_json, :to_h
+    alias_method :inspect, :to_s
+
+    def self.try_to_create_from_hash(payload)
+      return nil if payload.nil?
+      parsed_payload = try_parse_json(payload)
+      return payload unless parsed_payload
+      return payload unless parsed_payload.kind_of?(Hash)
+      hiwa = parsed_payload.with_indifferent_access
+      return payload unless (hiwa.has_key?(:status) && hiwa.has_key?(:body) && hiwa.has_key?(:error_message))
+      new(status: hiwa[:status], body: hiwa[:body], error_message: hiwa[:error_message])
+    end
+
+    def self.try_parse_json(str)
+      begin
+        JSON.parse(str)
+      rescue JSON::ParserError => e
+        return false
+      end
+    end
+  end
+end

data/lib/asynk/server.rb

@@ -0,0 +1,62 @@
+module Asynk
+  class Server
+    include Singleton
+
+    def initialize
+    end
+
+    def run
+      require 'asynk/worker'
+      prepare_consumers
+      register_signal_handlers
+      Asynk.logger.info "All consumers are prepared"
+      handle_signals
+      # handle_signals
+    end
+
+    def shutdown
+      futures = workers.map { |w| w.future.shutdown }
+      futures.map(&:value)
+      Asynk.broker.amqp_connection.close
+      Asynk.logger.info "Server shutdown!"
+    end
+
+    private
+      def handle_signals
+        loop do
+          signal = Thread.main[:signal_queue].shift
+          if signal
+            Asynk.logger.info "Caught sig#{signal.downcase}, stopping asynk server..."
+            shutdown
+            break
+          end
+          sleep(0.1)
+        end
+      end
+
+      def workers
+        @workers ||= []
+      end
+
+      def register_signal_handlers
+        Thread.main[:signal_queue] = []
+        %w(QUIT TERM INT).keep_if { |s| Signal.list.keys.include? s }.map(&:to_sym).each do |sig|
+          # This needs to be reentrant, so we queue up signals to be handled
+          # in the run loop, rather than acting on signals here
+          trap(sig) do
+            Thread.main[:signal_queue] << sig
+          end
+        end
+      end
+
+      def prepare_consumers
+        Asynk.consumers.each{ |consumer| prepare_consumer(consumer) }
+      end
+
+      def prepare_consumer(consumer)
+        consumer.concurrency.times do |index|
+          workers << Asynk::Worker.new(Asynk.broker.amqp_connection, consumer, index)
+        end
+      end
+  end
+end

data/lib/asynk/sync_publisher.rb

@@ -0,0 +1,36 @@
+module Asynk
+  class SyncPublisher
+    def initialize(routing_key, params)
+      @routing_key    = routing_key
+      @params         = params
+      @message_id     = (@params.delete(:message_id) || generate_message_id)
+      @wait_timeout   = (@params.delete(:timeout) || Asynk.config[:sync_publish_wait_timeout]) * 1000
+      @correlation_id = generate_message_id
+    end
+
+    def send
+      global_start_time = Asynk::Benchmark.start if Asynk.config[:publisher_execution_time]
+      Asynk.broker.pool.with do |channel, exchange, reply_queue|
+        exchange.publish(@params.to_json, message_id: @message_id, routing_key: @routing_key, correlation_id: @correlation_id, reply_to: reply_queue.name)
+
+        start_time = Asynk::Benchmark.start
+        while !@response do
+          delivery_info, properties, payload = reply_queue.pop
+          @response = payload if payload && properties[:correlation_id] == @correlation_id
+          raise(RuntimeError.new('Timeout error reached')) if @wait_timeout <= Asynk::Benchmark.end(start_time)
+        end
+      end
+
+      message = Asynk::Response.try_to_create_from_hash(@response)
+      if Asynk.config[:publisher_execution_time]
+        Asynk.logger.info "Sending sync message to #{@routing_key}:#{@message_id}. Completed In: #{Asynk::Benchmark.end(global_start_time)} ms."
+      end
+
+      message
+    end
+
+    def generate_message_id(legnth = 8)
+      SecureRandom.hex(legnth)
+    end
+  end
+end

data/lib/asynk/test_helper.rb

@@ -0,0 +1,52 @@
+module Asynk
+  module TestHelper
+    def publish_sync(routing_key, options)
+      consumers = find_consumers_by_route(routing_key)
+      raise "Cant find consumer by route: #{routing_key}" if consumers.empty?
+      raise "No ability to test multiple consumer per route" if consumers.count > 1
+      consumer = consumers.first
+
+      bunny_mocked_channel = BunnyMockedChannel.new
+      bunny_mocked_delivery_info = BuunyMockedDeliveryInfo.new(routing_key)
+      bunny_mock_properties = BuunyMockedProperties.new
+
+      message = Asynk::Message.new(bunny_mocked_delivery_info, bunny_mock_properties, options.to_json)
+      consumer_instance = consumer.new(bunny_mocked_channel, bunny_mocked_delivery_info) do |result|
+        self.asynk_response = result.kind_of?(String) ? result : result.to_json
+      end
+
+      consumer_instance.invoke_processing(message)
+      asynk_response
+    end
+
+    def asynk_response=(response)
+      @asynk_response = response
+    end
+
+    def asynk_response
+      @asynk_response
+    end
+
+    def find_consumers_by_route(route)
+      Asynk.consumers.select{ |consumer| consumer.routing_keys.any?{ |key| key == route } }
+    end
+
+    class BunnyMockedChannel
+      def ack(*args); end
+      def reject(*args); end
+      def retry(*args); end
+    end
+
+    class BuunyMockedDeliveryInfo
+      attr_reader :routing_key
+      def initialize(routing_key); @routing_key = routing_key; end
+      def exchange; end
+      def delivery_tag; end
+    end
+
+    class BuunyMockedProperties
+      def timestamp; Time.now; end
+      def message_id; 1; end
+    end
+  end
+end

data/lib/asynk/version.rb

@@ -0,0 +1,3 @@
+module Asynk
+  VERSION = "0.0.1"
+end