asynk 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b6ead08e1f2e9906b0871dd76ba54dd493c3873b
+  data.tar.gz: 872bbb7241da16c78dd5e54200d76f396852fd46
+SHA512:
+  metadata.gz: d60bf2cafe43ce3ff502459ec7d0ddb007e45257387eb3e3078c1af22bf0ad6dfc6f5103b5ed4f955a1a7fa16a7f96d933577bf243c3aa1d1306e656404b25f9
+  data.tar.gz: 30a07b0855bf08a7086914a9189f294ea61850fe74bde41868d605d58f1c806bd61880f0ea9cf5c84ce67083c3261265d2efa4fe95a3abef2fd09496502964ca

data/.gitignore

@@ -0,0 +1,18 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore the default SQLite database.
+/db/*.sqlite3
+/db/*.sqlite3-journal
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/log/.keep
+/tmp
+*.gem

data/Gemfile

@@ -0,0 +1,3 @@
+source 'http://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,172 @@
+PATH
+  remote: .
+  specs:
+    asynk (0.0.1)
+      activesupport
+      bunny
+      celluloid (~> 0.17.0)
+      celluloid-io
+      connection_pool
+      json (~> 1.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionmailer (4.2.3)
+      actionpack (= 4.2.3)
+      actionview (= 4.2.3)
+      activejob (= 4.2.3)
+      mail (~> 2.5, >= 2.5.4)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+    actionpack (4.2.3)
+      actionview (= 4.2.3)
+      activesupport (= 4.2.3)
+      rack (~> 1.6)
+      rack-test (~> 0.6.2)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    actionview (4.2.3)
+      activesupport (= 4.2.3)
+      builder (~> 3.1)
+      erubis (~> 2.7.0)
+      rails-dom-testing (~> 1.0, >= 1.0.5)
+      rails-html-sanitizer (~> 1.0, >= 1.0.2)
+    activejob (4.2.3)
+      activesupport (= 4.2.3)
+      globalid (>= 0.3.0)
+    activemodel (4.2.3)
+      activesupport (= 4.2.3)
+      builder (~> 3.1)
+    activerecord (4.2.3)
+      activemodel (= 4.2.3)
+      activesupport (= 4.2.3)
+      arel (~> 6.0)
+    activesupport (4.2.3)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    amq-protocol (2.0.0)
+    arel (6.0.3)
+    builder (3.2.2)
+    bunny (2.2.0)
+      amq-protocol (>= 2.0.0)
+    celluloid (0.17.1.2)
+      bundler
+      celluloid-essentials
+      celluloid-extras
+      celluloid-fsm
+      celluloid-pool
+      celluloid-supervision
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-essentials (0.20.2.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-extras (0.20.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-fsm (0.20.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-io (0.17.1)
+      bundler
+      celluloid (>= 0.17.1.1)
+      dotenv
+      nenv
+      nio4r (>= 1.1)
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-pool (0.20.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    celluloid-supervision (0.20.1.1)
+      bundler
+      dotenv
+      nenv
+      rspec-logsplit (>= 0.1.2)
+      timers (>= 4.1.1)
+    connection_pool (2.2.0)
+    dotenv (2.0.2)
+    erubis (2.7.0)
+    globalid (0.3.6)
+      activesupport (>= 4.1.0)
+    hitimes (1.2.2)
+    i18n (0.7.0)
+    json (1.8.3)
+    loofah (2.0.3)
+      nokogiri (>= 1.5.9)
+    mail (2.6.3)
+      mime-types (>= 1.16, < 3)
+    mime-types (2.6.1)
+    mini_portile (0.6.2)
+    minitest (5.8.0)
+    nenv (0.2.0)
+    nio4r (1.1.1)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    rack (1.6.4)
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rails (4.2.3)
+      actionmailer (= 4.2.3)
+      actionpack (= 4.2.3)
+      actionview (= 4.2.3)
+      activejob (= 4.2.3)
+      activemodel (= 4.2.3)
+      activerecord (= 4.2.3)
+      activesupport (= 4.2.3)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.2.3)
+      sprockets-rails
+    rails-deprecated_sanitizer (1.0.3)
+      activesupport (>= 4.2.0.alpha)
+    rails-dom-testing (1.0.7)
+      activesupport (>= 4.2.0.beta, < 5.0)
+      nokogiri (~> 1.6.0)
+      rails-deprecated_sanitizer (>= 1.0.1)
+    rails-html-sanitizer (1.0.2)
+      loofah (~> 2.0)
+    railties (4.2.3)
+      actionpack (= 4.2.3)
+      activesupport (= 4.2.3)
+      rake (>= 0.8.7)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.4.2)
+    rspec-logsplit (0.1.3)
+    sprockets (3.3.3)
+      rack (~> 1.0)
+    sprockets-rails (2.3.2)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (>= 2.8, < 4.0)
+    thor (0.19.1)
+    thread_safe (0.3.5)
+    timers (4.1.1)
+      hitimes
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  asynk!
+  minitest (~> 5.7, >= 5.7.0)
+  rails (~> 4)
+  rake (~> 10.0)

data/README.md

@@ -0,0 +1,168 @@
+# Asynk
+
+Asynk is a Ruby library for enabling synchronous/asynchronous inter-service communication, using RabbitMQ.
+
+# Overview
+
+It's takes concepts of ruby gem called Hutch, but using Cellouid under hood for creating workers for processing queues, which requires significant memory requirements.
+Also as limitation of ruby you cannot make heavy computations in ruby and gather true concurrency. Asynk offers synchronous calls (RPC).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'asynk'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install asynk
+
+## Usage
+
+
+Firstly you should define consumer, Example of consumer
+
+```ruby
+class V1::PaymentOrdersConsumer
+  include Asynk::Consumer  
+
+  set_consume 'sample_app.v1.users.create', 'sample_app.v1.users.notifications'
+
+  set_queue_options ack: true
+  set_subscribe_arguments manual_ack: true
+  set_concurrency 2
+  set_route_ending_as_action true  
+
+  # handling asynchronous request from amqp
+  def notifications(params)    
+    # do here some works
+  ensure
+    ack! # required if you manually processing acknowledgments, available methods reject!, requeue!
+  end
+
+  # example for handling synchronous request
+  def create(params)
+    result = my_method # here we doing some work, and saving result of
+    result = Asynk::Response.new(status: :ok, body: result) # result should any object which implements to_json.
+    respond(result) # if producer expecting response, we should response with some data.
+                    # Is not required to use any format of response. But preferred way is to use class Asynk::Response,
+                    # which used as lightweight implementation of http. (Containing body, status, and error_message).
+  ensure
+    ack!
+  end
+end
+```
+
+Firstly you should define Class, and include Asynk::Consumer.
+  * `set_consume` list of topics to consume
+  * `set_queue_options` set for options which created after server initialized
+  * `set_subscribe_arguments` set options which passed to when subscribing queue to exchange
+  * `set_concurrency` amount of workers which will consume for each consumer
+  * `set_route_ending_as_action` this options defines, that last item of consume topic is used as method name (ex: sample_app.v1.users.show, last item show, will be called in this class.)
+
+Also, after instance creation, has several methods,
+  * `log` logger object for sending logs for default Logger of Asynk.
+  * `ack!`, `reject!`, `requeue!`  methods is used for handling messages, if you want manually work with acknowledgments
+
+
+
+After declaring consumer, now you can send some request using default Asynk::Publisher, it has two options:
+  * `sync_publish` synchronous sending of message, and waiting for response from consumer. You should define timeouts, if consumers crashes it never receives a             message. timeout - says the amount of time in seconds for waiting message, if timeout reaches it sends TimeoutError.
+  * `publish` just sending message, and forgets about it.
+
+```ruby
+  # here sending synchronous messages
+  Asynk::Publisher.sync_publish('sample_app.v1.users.create', { name: 'Tom', surname: 'Lane', timeout: 10 })
+
+  # here sending asynchronous messages
+  Asynk::Publisher.publish('sample_app.v1.users.notifications', { name: 'Tom', surname: 'Lane' })
+```
+
+
+Asynk is using pool for channels for receiving and sending messages.
+Usage in rails, should initialize authentication data and init connection to RabbitMQ server.
+
+```ruby
+# Init authentication
+Asynk.config[:mq_host] = ENV['MQ_HOST']
+Asynk.config[:mq_username] = ENV['MQ_USERNAME']
+Asynk.config[:mq_password] = ENV['MQ_PASSWORD']
+
+# Init connection to server.
+Asynk::Broker.connect
+
+# here we initializing logger.
+if Rails.env.stage? || Rails.env.production?
+  if Asynk.booted_inside? # if we booting inside asynk, it's separate process than rails server.
+    Asynk.logger.level = ::Logger::DEBUG
+  else # If we booted inside  rails project, we can use rails's logger.
+    Asynk.logger = Rails.logger
+  end
+else
+  Asynk.logger.level = ::Logger::INFO
+  Asynk.config[:publisher_execution_time] = false
+end
+
+```
+## Config options
+* `mq_exchange` exchange to use for publishing (`default` 'asynk_exchange_topic')
+* `sync_publish_wait_timeout` time to wait for sync requests. If timeout reaches, the timeout raised. (`default` 10 seconds)
+* `default_consumer_concurrency` numbers of workers to start per consumer (`default` 1)
+* `logfile` log file for default logger of asynk (`default` 'log/asynk.log')
+* `pidifle`  Asynk consumers is running on different process, this file is used to store pid file (`default` 'tmp/pids/asynk.pid')
+* `mq_host`  host for connection to broker (RabbitMQ) (`default` 'localhost')
+* `mq_port`  port for connection to broker (RabbitMQ) (`default` 5672)
+* `mq_vhost` vhost for connection to broker (RabbitMQ) (`default` '/')
+* `mq_username` username for connection to broker (RabbitMQ) (`default` 'guest')
+* `mq_password` password for connection to broker (RabbitMQ) (`default` 'guest')
+* `publisher_execution_time` used for profiling time to send when using Asynk::Publisher (`default` true)
+* `respond_back_execution_time` used for profiling time used for processing sync response (`default` true)
+* `ignored_consumers` this parameter used for disabling unused consumers as array of strings with consumer class names(`default` [])
+
+
+# Testing your consumers
+Firstly you should include `Asynk::TestHelper` to your test class, and then call  `sync_publish` method for sending request, if this is rpc call,
+invoke the asynk_response method for getting response.
+
+Example using with Rails and MiniTest.
+```ruby
+  # test_helper.rb
+  class ActiveSupport::TestCase
+    # include the test helper.  
+    include Asynk::TestHelper
+
+    # wrapping the response with Asynk::Response class, otherwise it will be just string value.
+    def asynk_response
+      Asynk::Response.try_to_create_from_hash(super)
+    end
+  end
+
+
+  # some_consumer_test.rb  
+
+  test 'should show profile' do
+    publish_sync 'some_route', { name: 'Chris' }
+
+    assert asynk_response.success? # testing for status of the response
+    assert asynk_response[:unread_messages] # testing the returned data
+    assert asynk_response[:unread_message_count]    
+  end
+```
+
+## Known problems
+
+* Poor documentation (source are poorly documented)
+* Poor test coverage (there are almost no test)
+* RPC calls implementation. Currently is implemented as continues loop, which tries get data from reply queue. Before it was implemented using Mutex, which caused huge time usage on handling them. I am not sure that current implementation is correct, but is id much faster in current tests. (On my machine 1-2 ms vs 7-8 ms).
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/gm_server/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/asynk.gemspec

@@ -0,0 +1,27 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/asynk/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Danil Nurgaliev"]
+  gem.email         = ["jkonalegi@gmail.com"]
+  gem.summary       = "Async/sync inter sevrer communication tool."
+  gem.description   = "Async/sync inter sevrer communication tool, based on RabbitMQ and Celluloid"
+  gem.license       = "LGPL-3.0"
+  gem.homepage       = "https://github.com/konalegi/asynk"
+
+  gem.files         = `git ls-files -z`.split("\x0")
+  gem.executables   = gem.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "asynk"
+  gem.require_paths = ["lib"]
+  gem.version       = Asynk::VERSION
+  gem.add_dependency                  'celluloid', '~> 0.17'
+  gem.add_dependency                  'celluloid-io', '~> 0.17'
+  gem.add_dependency                  'connection_pool', '~> 2.2'
+  gem.add_dependency                  'activesupport', '~> 4.2'
+  gem.add_dependency                  'json', '~> 1.0'
+  gem.add_dependency                  'bunny', '~> 2.3'
+  gem.add_development_dependency      'minitest', '~> 5.7', '>= 5.7.0'
+  gem.add_development_dependency      'rake', '~> 10.0'
+  gem.add_development_dependency      'rails', '~> 4'
+end

data/bin/asynk

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/asynk/cli'
+
+begin
+  cli = Asynk::CLI.instance
+  cli.run
+rescue => e
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/bin/asynkctl

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+
+class Asynkctl
+  DEFAULT_KILL_TIMEOUT = 10
+
+  attr_reader :stage, :pidfile, :kill_timeout
+
+  def self.print_usage
+    puts "#{File.basename($0)} - stop a Asynk process from the command line."
+    puts
+    puts "Usage: #{File.basename($0)} <command> <pidfile> <kill_timeout>"
+    puts " where <command> is either 'quiet' or 'stop'"
+    puts "       <pidfile> is path to a pidfile"
+    puts "       <kill_timeout> is number of seconds to wait until Asynk exits"
+    puts "       (default: #{Asynkctl::DEFAULT_KILL_TIMEOUT}), after which Asynk will be KILL'd"
+    puts
+    puts "Be sure to set the kill_timeout LONGER than Asynk's -t timeout.  If you want"
+    puts "to wait 60 seconds for jobs to finish, use `Asynk -t 60` and `Asynkctl stop"
+    puts " path_to_pidfile 61`"
+    puts
+  end
+
+  def initialize(stage, pidfile, timeout)
+    @stage = stage
+    @pidfile = pidfile
+    @kill_timeout = timeout
+
+    done('No pidfile given', :error) if !pidfile
+    done("Pidfile #{pidfile} does not exist", :warn) if !File.exist?(pidfile)
+    done('Invalid pidfile content', :error) if pid == 0
+
+    fetch_process
+
+    begin
+      send(stage)
+    rescue NoMethodError
+      done "Invalid command: #{stage}", :error
+    end
+  end
+
+  def fetch_process
+    Process.getpgid(pid)
+  rescue Errno::ESRCH
+    done "Process doesn't exist", :error
+  end
+
+  def done(msg, error = nil)
+    puts msg
+    exit(exit_signal(error))
+  end
+
+  def exit_signal(error)
+    (error == :error) ? 1 : 0
+  end
+
+  def pid
+    @pid ||= File.read(pidfile).to_i
+  end
+
+  def quiet
+    Process.kill(:USR1, pid)
+  end
+
+  def stop
+    Process.kill(:TERM, pid)
+    kill_timeout.times do
+      begin
+        Process.getpgid(pid)
+      rescue Errno::ESRCH
+        FileUtils.rm_f pidfile
+        done 'Asynk shut down gracefully.'
+      end
+      sleep 1
+    end
+    Process.kill(:KILL, pid)
+    FileUtils.rm_f pidfile
+    done 'Asynk shut down forcefully.'
+  end
+  alias_method :shutdown, :stop
+end
+
+if ARGV.length < 2
+  Asynkctl.print_usage
+else
+  stage = ARGV[0]
+  pidfile = ARGV[1]
+  timeout = ARGV[2].to_i
+  timeout = Asynkctl::DEFAULT_KILL_TIMEOUT if timeout == 0
+
+  Asynkctl.new(stage, pidfile, timeout)
+end

data/consumer.rb

@@ -0,0 +1,52 @@
+require 'celluloid'
+require 'celluloid/io'
+require 'celluloid/autostart'
+require 'bunny'
+
+class Worker
+  include Celluloid::IO
+  finalizer :shutdown
+
+  def initialize(conn)
+    @ch = conn.create_channel
+    q = @ch.queue('task_queue', :durable => true)
+    q.subscribe manual_ack: true, &method(:on_event)
+    p 'worker accepting connections'
+  end
+
+  def on_event(delivery_info, properties, body)
+    p "#{Time.now.strftime('%FT %T.%L')} started work: #{body}"
+    sleep(10)
+    p "#{Time.now.strftime('%FT %T.%L')} completed work: #{body}"
+    @ch.ack(delivery_info.delivery_tag)
+  end
+
+  def shutdown
+    p 'trying to shutdown...'
+    @ch.close
+    p 'hey shutdown'
+  end
+end
+
+class Consumer
+  def initialize(options = {})
+    @size = (ENV['POOL'] || 2).to_i
+    @connection = Bunny.new(options)
+    @connection.start
+    @workers = @size.times.map{ Worker.new(@connection) }
+  end
+
+  def close_connection
+    futures = @workers.map { |w| w.future(:finalize) }
+    @connection.close if futures.all?
+  end
+end
+
+begin
+  @consumer = Consumer.new
+rescue Interrupt => _
+  @consumer.close_connection
+end
+
+sleep
+

data/lib/asynk/benchmark.rb

@@ -0,0 +1,21 @@
+module Asynk
+  class Benchmark
+    class << self
+
+      def measure_around(&block)
+        start_time = Time.now.to_f
+        block.call
+        ((Time.now.to_f - start_time)*1000).round(2)
+      end
+
+      def start
+        Time.now.to_f
+      end
+
+      def end(start_time)
+        ((Time.now.to_f - start_time)*1000).round(2)
+      end
+
+    end
+  end
+end

data/lib/asynk/broker.rb

@@ -0,0 +1,33 @@
+# require 'carrot-top'
+
+module Asynk
+  class Broker
+    class << self
+      def connect
+        @amqp_connection = Bunny.new(host: Asynk.config[:mq_host],
+                                     port: Asynk.config[:mq_port],
+                                 username: Asynk.config[:mq_username],
+                                 password: Asynk.config[:mq_password],
+                                    vhost: Asynk.config[:mq_vhost])
+        Asynk.logger.info [ "Connection to Rabbit with params host: #{Asynk.config[:mq_host]}:#{Asynk.config[:mq_port]}",
+                            "username: '#{Asynk.config[:mq_username]}' ", "vhost: '#{Asynk.config[:mq_vhost]}'"
+                          ].join(' ')
+
+        @amqp_connection.start
+
+        @pool = ConnectionPool.new(size: 10, timeout: 5) do
+          channel = @amqp_connection.create_channel(nil, nil)
+          [channel, channel.topic(Asynk.config[:mq_exchange]), channel.queue('', exclusive: true)]
+        end
+      end
+
+      def disconnect
+        @amqp_connection.close if @amqp_connection
+        @amqp_connection = nil
+      end
+
+      def pool; @pool; end
+      def amqp_connection; @amqp_connection; end
+    end
+  end
+end