kinetic 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7a08f594cae0d5c01c00f340386df7c7788a745
-  data.tar.gz: 280fe43699041a8a233b455887d795b6c47e42f8
+  metadata.gz: f720debb00d4ab62ce3b3e8d7e4f7eaf422be407
+  data.tar.gz: a01b254919820ca788f0aa9ee8d42c06a7a7ca06
 SHA512:
-  metadata.gz: 65871da55313e959082177229600037b1ffdf5e94b1886bebb525e9099effe4b249fb7b581f54d45ed908fa68eac60e0ce7e44841e96f0738d7ae1fc731a8c3f
-  data.tar.gz: e8452cb376cab92b21796269deff0ac5ab24306099675eca52eeb9d64071cced80ba649e0b6399748f77d113b13e6bdd7f7aecbcf1a4bc1cbf386faa30be73ff
+  metadata.gz: 6243b0b4ae9062a16aeda4bf06626088fd0cfaf80ed2de1b4a385f827ff491c063a69e168ef90cd46d35ba8f6b0ca0a1909a261c99670dd38634b730ede822d2
+  data.tar.gz: be6f7059f4fb70d65e6d1cab62357d9f082f783a249e3e7e40694d93de2a7cf71b182795d26d184d26588ab524d93d2aab6b0c4535723013a095cfcf4fbae731

data/README.md

@@ -14,19 +14,53 @@ Kinetic is an AMQP worker framework designed in vein of microframeworks such as
 gem install kinetic
 ```
 
-### Basic Usage
+## Basic Usage
 
 Kinetic follows a pattern similar to that of Sinatra, so most of the conventions should be recognizable.
 
 
 Example: simple_worker.rb
 
+```ruby
+on 'message' do |message|
+    puts "Received #{message}"
+end
 ```
+
+This code will create a queue on the amqp server listening for messages on the simple_worker.rb.direct exchange with
+the routing key 'message'. By default the message is expected to be in JSON format, however in the future other
+serializers may be available.
+
+## Configuration
+Kinetic configuration follows a similar DSL to that of sinatra. Configuration can be set by calling the `set` method,
+and can be retrieved by referencing the `config` variable or the `get` or `get!` methods.
+
+### set
+To set a configuration value use set:
+
+```ruby
 set :host, 'localhost'
-set :port, 5678
 
 on 'message' do |message|
     puts "Received #{message}"
 end
 ```
 
+### Defaults
+[Kinetic::Configuration](http://rdoc.info/github/fugufish/kinetic/master/Kinetic/Configuration)
+
+
+
+## Changelog
+
+### 0.0.3
+* Fix bug where process shut down does not kill children
+* Add ability to set process name for master and workers
+* Add after fork block capability allowing code to be written to run after the worker forks
+
+### 0.0.1
+* Initial release
+
+
+
+

data/lib/kinetic/base.rb

@@ -1,6 +1,7 @@
 # ruby dependencies
 require 'logger'
 require 'fileutils'
+require 'yaml'
 
 # external dependencies
 require 'amqp'
@@ -8,6 +9,7 @@ require 'confstruct'
 require 'kgio'
 require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/hash/reverse_merge'
+require 'active_support/inflections'
 require 'awesome_print'
 require 'awesome_print/core_ext/logger'
 require 'msgpack'
@@ -24,7 +26,7 @@ module Kinetic
 
   module Delegator
 
-    %w{on config logging set}.each { |m| delegate m.to_sym, to: :application }
+    %w{on config logging set config_file deserialize}.each { |m| delegate m.to_sym, to: :application }
 
     def application
       Application
@@ -38,22 +40,38 @@ module Kinetic
     class << self
       include Kinetic::DSL
 
+
+      def after_fork_procs
+        @after_fork_procs ||= []
+      end
+
       def exchanges
         @exchanges ||= {}
       end
 
+      def deserialize(message)
+        @serializer.deserialize(message)
+      end
+
       private
 
+      def set_serializer(serializer)
+        logger.debug "Loading serializer #{serializer}"
+        require "kinetic/serializers/#{serializer.downcase}"
+        @serializer = Object.const_get("Kinetic::Serializers::#{serializer}")
+      end
+
       def direct
         exchanges[:direct] ||= {}
       end
 
+    end
 
-
-
+    def initialize
+      self.class.send(:set_serializer, config.serializer)
     end
 
-    delegate :logger, :set, :get, :get!, :exchanges,  to: :class
+    delegate :logger, :set, :get, :get!, :exchanges, :after_fork_procs, :deserialize, :config, to: :class
 
   end
 

data/lib/kinetic/cli.rb

@@ -9,13 +9,14 @@ module Kinetic
     end
 
     def self.parse
-      if ARGV[0] && COMMANDS.include?(ARGV[0])
+      command = nil
+      if ARGV.empty? || (ARGV[0] && COMMANDS.include?(ARGV[0]))
         command = ARGV.shift
       else
         puts PARSER.banner
-        exit 1
+        exit! 1
       end
-      command
+      command || 'run'
     end
 
   end
@@ -24,7 +25,7 @@ module Kinetic
   end
 
 
-  at_exit { Master.new(Application.new).send(CLI.parse) }
+  at_exit { Master.new(Application.new).send(CLI.parse) if $!.nil? }
 
 end
 

data/lib/kinetic/configuration.rb

@@ -7,11 +7,15 @@ module Kinetic
       app_file: $0,
       workers:  1,
       log_file: STDOUT,
+      host: 'localhost',
+      port: 5672,
+      serializer: :JSON,
+      timeout: 1
     }
 
     def initialize
       super
-      self.reverse_merge!(DEFAULTS)
+      self.deep_merge!(DEFAULTS)
     end
 
   end

data/lib/kinetic/dsl.rb

@@ -2,6 +2,11 @@ module Kinetic
 
   module DSL
 
+    def after_fork(&block)
+      raise Kinetic::Errors::BlockMissing unless block_given?
+      after_fork_procs << block
+    end
+
     # Sets a configuration value
     #
     # @param [Symbol] key the configuration key to set
@@ -9,10 +14,19 @@ module Kinetic
     #
     # @return [Boolean] returns true
     def set(key, value)
+      logger.debug "Setting '#{key}' to '#{value}'"
       config[key.to_sym] = value
       true
     end
 
+    # Set configuration file.
+    #
+    # @param [String] path to configuration file
+    def config_file(file)
+      set :config_file, File.expand_path(file)
+      config.configure(YAML.load(File.new(get!(:config_file), 'r')))
+    end
+
     # Gets a configuration value
     #
     # @param [Symbol] key the key of the value to get
@@ -57,12 +71,12 @@ module Kinetic
       @logger ||= reopen_logger
     end
 
-    private
-
     def config
       @config ||= Kinetic::Configuration.new
     end
 
+    protected
+
     def reopen_logger
       @logger = Logger.new(config.log_file)
     end

data/lib/kinetic/errors.rb

@@ -3,7 +3,8 @@ module Kinetic
 
     class KineticError < StandardError; end
 
-    class NoSubscriberBlock < KineticError; end
+    class BlockMissing      < KineticError; end
+    class NoSubscriberBlock < BlockMissing; end
     class KeyMustBeString   < KineticError; end
 
     class MissingConfigurationValue < KineticError

data/lib/kinetic/master.rb

@@ -7,10 +7,10 @@ module Kinetic
     WORKERS    = {}
     SIG_QUEUE  = []
 
-    delegate :logger, :get, :get!, :set, to: :app
+    delegate :logger, :get, :get!, :set, :after_fork_procs, to: :app
 
 
-    attr_reader :app
+    attr_reader :app, :reexecpid
 
     def initialize(app)
       @app = app
@@ -24,6 +24,7 @@ module Kinetic
         logger.info "Starting Kinetic #{Kinetic::VERSION} with PID #{Process.pid}"
         logger.debug 'Configuration:'
         logger.ap    app.class.send(:config).to_hash
+        proc_name 'master'
         write_pidfile!
         initialize_self_pipe!
         initialize_signal_traps!
@@ -38,22 +39,6 @@ module Kinetic
       end
     end
 
-    # Sets or gets the before fork block. This is called just before a worker is forked.
-    #
-    # @return [Proc] the before_block
-    def before_fork(&block)
-      @before_fork = block if block_given?
-      @before_fork || proc {}
-    end
-
-    # Sets or gets the after fork block. This is called immediately after a worker is forked.
-    #
-    # @return [Proc] the after_fork block
-    def after_fork(&block)
-      @after_fork = block if block_given?
-      @after_fork || proc {}
-    end
-
     private
 
     def master_sleep(time)
@@ -67,13 +52,11 @@ module Kinetic
       when :QUIT
         break
       when :TERM, :INT
-        logger.warn "#{get(:name)} is shutting down immediately!"
-        exit!(1)
+        stop(false)
         break
       else
         master_sleep(0.5)
       end while true
-      logger.warn "#{get(:name)} is going down!"
     end
 
     # Based on Unicorn's self-pipe
@@ -110,12 +93,13 @@ module Kinetic
         WORKERS.value?(worker_nr) and next
         #noinspection RubyArgCount
         worker = Worker.new(worker_nr, @app)
-        logger.debug 'Calling before_fork block'
-        before_fork.call(self, worker)
         logger.debug 'Forking worker'
         parent = Process.pid
         pid = fork do
           begin
+            proc_name "worker-#{worker_nr}"
+            logger.debug 'Calling after fork procs'
+            after_fork_procs.each { |p| p.call }
             #noinspection RubyArgCount
             worker.run
           rescue => e
@@ -125,6 +109,7 @@ module Kinetic
           end
         end
         logger.debug "Worker #{worker_nr} started on #{pid}"
+        WORKERS[pid] = worker
       end
     rescue => e
       logger.error(e)
@@ -148,7 +133,59 @@ module Kinetic
       WORKERS.clear
 
       after_fork.call(self, worker)
+    end
+
+    # Terminates all workers, but does not exit master process
+    def stop(graceful = true)
+      logger.warn graceful ? 'Process is shutting down gracefully' : 'Process is shutting down immediately!'
+      limit = Time.now + get!(:timeout)
+      until WORKERS.empty? || Time.now > limit
+        if graceful
+          soft_kill_each_worker(:QUIT)
+        else
+          kill_each_worker(:TERM)
+        end
+        sleep(0.1)
+        reap_all_workers
+      end
+      kill_each_worker(:KILL)
+    end
+
+    def kill_each_worker(signal)
+      WORKERS.keys.each { |wpid| kill_worker(signal, wpid) }
+    end
+
+    # delivers a signal to a worker and fails gracefully if the worker
+    # is no longer running.
+    def kill_worker(signal, wpid)
+      Process.kill(signal, wpid)
+    rescue Errno::ESRCH
+      worker = WORKERS.delete(wpid) and worker.close rescue nil
+    end
+
+    # reaps all unreaped workers
+    def reap_all_workers
+
+      begin
+        wpid, status = Process.waitpid2(-1, Process::WNOHANG)
+        wpid or return
+        if reexec_pid == wpid
+          logger.error "reaped #{status.inspect} exec()-ed"
+          self.reexec_pid = 0
+          self.pid = pid.chomp('.oldbin') if pid
+          proc_name 'master'
+        else
+          worker = WORKERS.delete(wpid) and worker.close rescue nil
+          m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
+          status.success? ? logger.info(m) : logger.error(m)
+        end
+      rescue Errno::ECHILD
+        break
+      end while true
+    end
 
+    def proc_name(tag)
+      $0 = "#{get!(:name)} [#{tag}]"
     end
 
   end

data/lib/kinetic/serializers/json.rb

@@ -0,0 +1,19 @@
+require 'json'
+
+module Kinetic
+  module Serializers
+
+    module JSON
+
+      def self.serialize(message)
+        ::JSON.generate(message)
+      end
+
+      def self.deserialize(message)
+        ::JSON.parse(message)
+      end
+
+    end
+
+  end
+end

data/lib/kinetic/version.rb

@@ -1,3 +1,3 @@
 module Kinetic
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

data/lib/kinetic/worker.rb

@@ -1,7 +1,7 @@
 module Kinetic
   class Worker
 
-    delegate :logger, :get, :get!, :set, to: :app
+    delegate :logger, :get, :get!, :set, :deserialize, to: :app
 
     attr_reader :id, :app, :channel, :exchanges
 
@@ -61,7 +61,11 @@ module Kinetic
           q.bind(exchanges[name], routing_key: queue)
           logger.debug "   Subscribing to messages for '#{queue}'"
           q.subscribe do |meta, payload|
-            block.call(MessagePack.unpack(payload))
+            begin
+              block.call(deserialize(payload))
+            rescue Exception => e
+              logger.error e
+            end
           end
         end
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kinetic
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Jarod Reid
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2014-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: confstruct
@@ -187,6 +187,7 @@ files:
 - lib/kinetic/errors.rb
 - lib/kinetic/master.rb
 - lib/kinetic/publisher.rb
+- lib/kinetic/serializers/json.rb
 - lib/kinetic/version.rb
 - lib/kinetic/worker.rb
 - spec/base_spec.rb