kinetic 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 67a4ff614b5f65c94ff43ae5e18e7a73944c13e3
+  data.tar.gz: 41f13ae0b524573c98a63f853d7be3c458c12a96
+SHA512:
+  metadata.gz: 72ce1e7b862c33785117568f17944f7d40401c158e78e80a3a44e7b865d299ef1c521c24104a81b666397689e069273434be0cbaa553f5fb6f83b7a6411c4625
+  data.tar.gz: 575a3a38d19d22d92125da043f9a065cd18f53d348003f3eda1ac9ef0679476ed636f923bc0a2d722d6c16e3dd08d87df3f18a87f532869463f1f3f50cb8f1be

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea/**
+*.iml

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in kinetic.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Jarod Reid
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,28 @@
+# Kinetic
+
+Kinetic is an AMQP worker framework designed in vein of microframeworks such as sinatra. Its goal is to provide an
+ easy way to configure and run AMQP consumers in order to reduce developer overhead when working with AMQP. Kinetic
+ follows the master/worker convention used in Unicorn, and its prefork code is mostly based on that of Unicorn's.
+
+## Installation
+
+```ruby
+gem install kinetic
+```
+
+### Basic Usage
+
+Kinetic follows a pattern similar to that of Sinatra, so most of the conventions should be recognizable.
+
+
+Example: simple_worker.rb
+
+```
+set :host, 'localhost'
+set :port, 5678
+
+on 'message' do |message|
+    puts "Received #{message}"
+end
+```
+

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/examples/simple_worker.rb

@@ -0,0 +1,8 @@
+require_relative '../lib/kinetic'
+
+set :host, '127.0.0.1'
+set :port, 5672
+
+on('message') do |message|
+  puts message
+end

data/kinetic.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'kinetic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'kinetic'
+  spec.version       = Kinetic::VERSION
+  spec.authors       = ['Jarod Reid']
+  spec.email         = ['jreid@voyst.com']
+  spec.description   = %q{A powerful yet simple AMQP worker framework}
+  spec.summary       = %q{A powerful yet simple AMQP worker framework}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'confstruct'
+  spec.add_dependency 'amqp'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'kgio'
+  spec.add_dependency 'awesome_print'
+  spec.add_dependency 'msgpack'
+  spec.add_dependency 'bunny'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+
+
+
+end

data/lib/kinetic.rb

@@ -0,0 +1,3 @@
+require_relative './kinetic/cli'
+
+

data/lib/kinetic/base.rb

@@ -0,0 +1,87 @@
+# ruby dependencies
+require 'logger'
+require 'fileutils'
+
+# external dependencies
+require 'amqp'
+require 'confstruct'
+require 'kgio'
+require 'active_support/core_ext/module/delegation'
+require 'active_support/core_ext/hash/reverse_merge'
+require 'awesome_print'
+require 'awesome_print/core_ext/logger'
+require 'msgpack'
+
+# kinetic dependencies
+require_relative '../kinetic/version'
+require_relative '../kinetic/configuration'
+require_relative '../kinetic/master'
+require_relative '../kinetic/worker'
+
+module Kinetic
+
+  module Delegator
+
+    %w{on config logging set}.each { |m| delegate m.to_sym, to: :application }
+
+    def application
+      Application
+    end
+
+
+  end
+
+  class Base
+
+    class << self
+
+      attr_reader :exchanges
+
+      def set(key, value)
+        config[key] = value
+      end
+
+      def config
+        @config ||= Kinetic::Configuration.new
+      end
+
+      def logger
+        @logger ||= reopen_logger
+      end
+
+      def reopen_logger
+        @logger = Logger.new(STDOUT)
+      end
+
+      def on(key, &block)
+        logger.debug "Setting up '#{key}' on 'direct'"
+        direct[key] = block
+      end
+
+      private
+
+      def direct
+        exchanges[:direct] ||= {}
+      end
+
+
+      def exchanges
+        @exchanges ||= {}
+      end
+
+    end
+
+    delegate :logger, to: :class
+    delegate :config, to: :class
+
+    def reopen_logger
+      self.class.reopen_logger
+    end
+
+    def exchanges
+      self.class.send(:exchanges)
+    end
+
+  end
+
+end

data/lib/kinetic/cli.rb

@@ -0,0 +1,31 @@
+require 'kinetic/base'
+module Kinetic
+  module CLI
+
+    COMMANDS = %w{run start stop}
+
+    PARSER = OptionParser.new do |opts|
+      opts.banner = "Usage #{File.basename($0)} (#{COMMANDS.join('|')}) [options]"
+    end
+
+    def self.parse
+      if ARGV[0] && COMMANDS.include?(ARGV[0])
+        command = ARGV.shift
+      else
+        puts PARSER.banner
+        exit 1
+      end
+      command
+    end
+
+  end
+
+  class Application < Base
+  end
+
+
+  at_exit { Master.new(Application.new).send(CLI.parse) }
+
+end
+
+extend Kinetic::Delegator

data/lib/kinetic/configuration.rb

@@ -0,0 +1,17 @@
+module Kinetic
+  class Configuration < Confstruct::Configuration
+
+    DEFAULTS = {
+      name:     File.basename($0),
+      root:     File.dirname($0),
+      app_file: $0,
+      workers:  1,
+    }
+
+    def initialize
+      super
+      self.reverse_merge!(DEFAULTS)
+    end
+
+  end
+end

data/lib/kinetic/errors.rb

@@ -0,0 +1,18 @@
+module Kinetic
+  module Errors
+
+    class KineticError < StandardError; end
+
+    class ConnectionError < KineticError
+
+      def initialize(error)
+        message   = error.message
+        backtrace = error.backtrace
+        super message
+        self.set_backtrace(backtrace)
+      end
+
+    end
+
+  end
+end

data/lib/kinetic/master.rb

@@ -0,0 +1,132 @@
+require 'optparse'
+module Kinetic
+  class Master
+
+    SELF_PIPE  = []
+    QUEUE_SIGS = [ :WINCH, :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, :TTIN, :TTOU ]
+    WORKERS    = {}
+    SIG_QUEUE  = []
+
+    delegate :logger, to: :app
+    delegate :config, to: :app
+
+    attr_reader :app
+
+    def initialize(app)
+      @app = app
+    end
+
+    # Starts and initializes the master process.
+    #
+    # @return [Kinetic::Master] the master process
+    def run
+      begin
+        logger.info "Starting Kinetic #{Kinetic::VERSION} with PID #{Process.pid}"
+        logger.debug 'Configuration:'
+        logger.ap    config.to_hash
+        write_pidfile!
+        initialize_self_pipe!
+        initialize_signal_traps!
+        spawn_missing_workers
+        join
+      ensure
+        call_on_exit_callbacks
+        exit!
+      end
+    end
+
+    # Sets or gets the before fork block. This is called just before a worker is forked.
+    #
+    # @return [Proc] the before_block
+    def before_fork(&block)
+      @before_fork = block if block_given?
+      @before_fork || proc {}
+    end
+
+    # Sets or gets the after fork block. This is called immediately after a worker is forked.
+    #
+    # @return [Proc] the after_fork block
+    def after_fork(&block)
+      @after_fork = block if block_given?
+      @after_fork || proc {}
+    end
+
+    private
+
+    def join
+      logger.debug "joining thread"
+      loop do
+        sleep 1
+      end
+    end
+
+    # Based on Unicorn's self-pipe
+    def initialize_self_pipe!
+      logger.debug 'Initializing self pipe'
+      SELF_PIPE.replace Kgio::Pipe.new.each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+    end
+
+    def initialize_signal_traps!
+      logger.debug 'Initializing signal traps'
+      QUEUE_SIGS.each { |sig| trap(sig) {SIG_QUEUE << sig; awaken_master } }
+      trap(:CHLD) { awaken_master }
+    end
+
+    def awaken_master
+      logger.debug 'Awakening master'
+      SELF_PIPE[1].kgio_trywrite('.')
+    end
+
+    def write_pidfile!
+      config.pid ||= File.join(config.root, "#{config.name}.pid")
+      logger.info "Writing PID file to #{config.pid}"
+      File.open(config.pid, 'w') { |f| f.write(Process.pid) }
+    end
+
+    def call_on_exit_callbacks
+    ensure
+      FileUtils.rm(config.pid)
+    end
+
+    def spawn_missing_workers
+      logger.debug 'Spawning missing workers'
+      worker_nr = -1
+      until (worker_nr += 1) == config.workers
+        WORKERS.value?(worker_nr) and next
+        worker = Worker.new(worker_nr, @app)
+        logger.debug 'Calling before_fork block'
+        before_fork.call(self, worker)
+        logger.debug 'Forking worker'
+        pid = fork do
+          app.reopen_logger
+          worker.run
+        end
+        logger.debug "Worker #{worker_nr} started on #{pid}"
+      end
+    rescue => e
+      logger.error(e)
+    end
+
+    EXIT_SIGS         = [ :QUIT, :TERM, :INT ]
+    WORKER_QUEUE_SIGS = QUEUE_SIGS - EXIT_SIGS
+
+    # gets rid of stuff the worker has no business keeping track of
+    # to free some resources and drops all sig handlers.
+    # traps for USR1, USR2, and HUP may be set in the after_fork Proc
+    # by the user.
+    def init_worker_process(worker)
+      worker.atfork_child
+      # we'll re-trap :QUIT later for graceful shutdown iff we accept clients
+      EXIT_SIGS.each { |sig| trap(sig) { exit!(0) } }
+      exit!(0) if (SIG_QUEUE & EXIT_SIGS)[0]
+      WORKER_QUEUE_SIGS.each { |sig| trap(sig, nil) }
+      trap(:CHLD, 'DEFAULT')
+      SIG_QUEUE.clear
+      WORKERS.clear
+
+      after_fork.call(self, worker)
+
+    end
+
+  end
+end

data/lib/kinetic/publisher.rb

@@ -0,0 +1,6 @@
+require 'logger'
+
+require 'msgpack'
+require 'bunny'
+
+#TODO: Finish publisher

data/lib/kinetic/version.rb

@@ -0,0 +1,3 @@
+module Kinetic
+  VERSION = "0.0.1"
+end

data/lib/kinetic/worker.rb

@@ -0,0 +1,73 @@
+module Kinetic
+  class Worker
+
+    delegate :logger, to: :app
+    delegate :config, to: :app
+
+    attr_reader :id, :app, :channel, :exchanges
+
+    def initialize(id, app)
+      @app = app
+      logger.debug "Initializing worker #{id}"
+      @id = id
+      @to_io, @master = Kgio::Pipe.new.each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
+    end
+
+    def atfork_parent
+      logger.debug "Worker #{id} closing pipe to child"
+      @to_io = @to_io.close
+    end
+
+    def atfork_child
+      logger.debug "Worker #{id} closing pipe to master"
+      @master = @master.close
+    end
+
+    def run
+      logger.debug "Establishing connection host: '#{config.host}', port: '#{config.port}'"
+      AMQP.start(host: config.host, port: config.port) do |connection|
+        app.reopen_logger
+        logger.debug "AMQP started with conneciton #{connection}"
+        initialize_channel!(connection)
+        initialize_exchanges!
+        initialize_subscribers!
+      end
+    end
+
+
+    private
+
+    def initialize_channel!(connection)
+      logger.debug 'Intitializing channel'
+      @channel = AMQP::Channel.new(connection)
+    end
+
+    def initialize_exchanges!
+      logger.debug 'Initializing exchanges'
+      @exchanges = {}
+      prefix = config.name
+      app.exchanges.each_key do |name|
+        logger.debug "   Initializing #{name} exchange"
+        @exchanges[name] = channel.send(name, "#{prefix}.#{name}")
+      end
+    end
+
+    def initialize_subscribers!
+      logger.debug 'Initializing subscribers'
+      app.exchanges.each do |name, value|
+        logger.debug "   Building subscribers for '#{name}' exchange"
+        value.each do |queue, block|
+          logger.debug "   Initializing queue for '#{queue}'"
+          q = channel.queue(queue)
+          logger.debug "   Binding queue to '#{name}' exchange"
+          q.bind(exchanges[name], routing_key: queue)
+          logger.debug "   Subscribing to messages for '#{queue}'"
+          q.subscribe do |meta, payload|
+            block.call(MessagePack.unpack(payload))
+          end
+        end
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,187 @@
+--- !ruby/object:Gem::Specification
+name: kinetic
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jarod Reid
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: confstruct
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kgio
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A powerful yet simple AMQP worker framework
+email:
+- jreid@voyst.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- examples/simple_worker.rb
+- kinetic.gemspec
+- lib/kinetic.rb
+- lib/kinetic/base.rb
+- lib/kinetic/cli.rb
+- lib/kinetic/configuration.rb
+- lib/kinetic/errors.rb
+- lib/kinetic/master.rb
+- lib/kinetic/publisher.rb
+- lib/kinetic/version.rb
+- lib/kinetic/worker.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: A powerful yet simple AMQP worker framework
+test_files: []
+has_rdoc: