carnivore 0.1.0

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+# v0.1.0
+* Initial release

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,64 @@
+PATH
+  remote: .
+  specs:
+    carnivore (0.1.0)
+      bunny
+      celluloid
+      fog
+      mixlib-config
+      reel
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    builder (3.2.2)
+    bunny (0.8.0)
+    celluloid (0.15.1)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
+    certified (0.1.1)
+    excon (0.25.3)
+    fog (1.15.0)
+      builder
+      excon (~> 0.25.0)
+      formatador (~> 0.2.0)
+      mime-types
+      multi_json (~> 1.0)
+      net-scp (~> 1.1)
+      net-ssh (>= 2.1.3)
+      nokogiri (~> 1.5)
+      ruby-hmac
+    formatador (0.2.4)
+    http (0.4.0)
+      certified
+      http_parser.rb
+    http_parser.rb (0.5.3)
+    mime-types (1.25)
+    mini_portile (0.5.1)
+    mixlib-config (1.1.2)
+    multi_json (1.8.0)
+    net-scp (1.1.2)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.6.8)
+    nio4r (0.5.0)
+    nokogiri (1.6.0)
+      mini_portile (~> 0.5.0)
+    rack (1.5.2)
+    reel (0.3.0)
+      celluloid-io (>= 0.8.0)
+      http (>= 0.2.0)
+      http_parser.rb (>= 0.5.3)
+      rack (>= 1.4.0)
+      websocket_parser (>= 0.1.0)
+    ruby-hmac (0.4.0)
+    timers (1.1.0)
+    websocket_parser (0.1.4)
+      http
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  carnivore!

data/README.md

@@ -0,0 +1,59 @@
+# Carnivore
+
+Eat messages, rule the world.
+
+## Purpose
+
+Slim library to consume messages. Sources are defined
+and callbacks are registered to defined sources. Sources
+feed messages to callback workers asynchronously and
+stuff gets done. Super simple!
+
+## Usage
+
+1. Build a source
+2. Add callbacks
+3. Profit!
+
+```ruby
+Carnivore.configure do
+  src = Source.build(:type => :test, :args => {})
+  src.add_callback(:print_message) do |msg|
+    puts "Received message: #{message}"
+  end
+end.start!
+```
+
+### Advanced Usage
+
+Under the hood, callbacks are built into `Carnivore::Callback`
+instances. This class can be subclassed and provided directly
+instead of a simple block. This has the added bonus of being
+able to define the number of worker instances to be created
+for the callback (blocks default to 1):
+
+```ruby
+require 'carnivore'
+
+class CustomCallback < Carnivore::Callback
+
+  self.workers = 5
+
+  def setup
+    info "Custom callback setup called!"
+  end
+
+  def execute(message)
+    info "GOT MESSAGE: #{message[:message]} - source: #{message[:source]} - instance: #{self}"
+  end
+end
+
+Carnivore.configure do
+  s = Carnivore::Source.build(:type => :test, :args => {})
+  s.add_callback(:printer, CustomCallback)
+end.start!
+```
+
+## Info
+
+* Repository: https://github.com/heavywater/carnivore

data/carnivore.gemspec

@@ -0,0 +1,18 @@
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__)) + '/lib/'
+require 'carnivore/version'
+Gem::Specification.new do |s|
+  s.name = 'carnivore'
+  s.version = Carnivore::VERSION.version
+  s.summary = 'Message processing helper'
+  s.author = 'Chris Roberts'
+  s.email = 'chrisroberts.code@gmail.com'
+  s.homepage = 'https://github.com/heavywater/carnivore'
+  s.description = 'Message processing helper'
+  s.require_path = 'lib'
+  s.add_dependency 'fog'
+  s.add_dependency 'celluloid'
+  s.add_dependency 'reel'
+  s.add_dependency 'mixlib-config'
+  s.add_dependency 'bunny'
+  s.files = Dir['**/*']
+end

data/examples/test_block.rb

@@ -0,0 +1,10 @@
+require 'carnivore'
+
+Carnivore.configure do
+  s = Carnivore::Source.build(:type => :test, :args => {})
+
+  s.add_callback(:printer) do |message|
+    info "GOT MESSAGE: #{message[:message]} - source: #{message[:source]} - instance: #{self}"
+  end
+
+end.start!

data/examples/test_class.rb

@@ -0,0 +1,20 @@
+require 'carnivore'
+
+class CustomCallback < Carnivore::Callback
+
+  self.workers = 5
+
+  def setup
+    info "Custom callback setup called!"
+  end
+
+  def execute(message)
+    info "GOT MESSAGE: #{message[:message]} - source: #{message[:source]} - instance: #{self}"
+  end
+end
+
+Carnivore.configure do
+  s = Carnivore::Source.build(:type => :test, :args => {})
+  s.add_callback(:printer, CustomCallback)
+
+end.start!

data/examples/test_http.rb

@@ -0,0 +1,10 @@
+require 'carnivore'
+
+Carnivore.configure do
+  s = Carnivore::Source.build(:type => :http, :args => {:bind => '0.0.0.0', :port => 3000})
+
+  s.add_callback(:printer) do |message|
+    info "GOT MESSAGE: #{message[:message][:body]} - path: #{message[:message][:request].url} - method: #{message[:message][:request].method} - source: #{message[:source]} - instance: #{self}"
+  end
+
+end.start!

data/lib/carnivore/callback.rb

@@ -0,0 +1,48 @@
+module Carnivore
+  class Callback
+
+    class << self
+      attr_accessor :workers
+    end
+
+    include Celluloid
+    include Celluloid::Logger
+
+    execute_block_on_receiver :execute
+
+    attr_reader :name
+
+    def initialize(name, block=nil)
+      @name = name
+      @block = block
+      if(@block.nil? && self.class == Callback)
+        raise ArgumentError.new 'Block is required for dynamic callbacks!'
+      end
+      setup
+    end
+
+    def setup
+    end
+
+    def inspect
+      "callback<#{self.object_id}>"
+    end
+
+    def valid?(message)
+      true
+    end
+
+    def call(message)
+      raise TypeError.new('Invalid message for this callback!') unless valid?(message)
+      @block ? execute(message, &@block) : execute(message)
+    rescue => e
+      error "[callback: #{self}, source: #{message[:source]}, message: #{message[:message].object_id}]: #{e.class} - #{e}"
+      debug "#{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+    end
+
+    def execute(message)
+      yield message
+    end
+
+  end
+end

data/lib/carnivore/config.rb

@@ -0,0 +1,42 @@
+require 'json'
+require 'mixlib/config'
+
+module Carnivore
+  class Config
+
+    extend Mixlib::Config
+
+    class << self
+
+      def configure(args)
+        build(args[:config_path]) if args[:config_path]
+        self.merge!(args)
+        self
+      end
+
+      def build(path_or_hash)
+        if(path_or_hash.is_a?(Hash))
+          conf = path_or_hash
+        else
+          if(File.exists?(path_or_hash.to_s))
+            conf = JSON.load(File.read(path_or_hash))
+            self.config_path = path_or_hash
+          else
+            raise "Failed to load configuration file: #{path_or_hash}"
+          end
+        end
+        conf.each do |k,v|
+          self.send(k, v)
+        end
+        self
+      end
+
+      def get(*ary)
+        ary.flatten.inject(self) do |memo, key|
+          memo[key.to_s] || memo[key.to_sym] || break
+        end
+      end
+
+    end
+  end
+end

data/lib/carnivore/container.rb

@@ -0,0 +1,17 @@
+require 'celluloid/logger'
+
+module Carnivore
+  class Container < Module
+
+    include Celluloid::Logger
+
+    class << self
+      def log
+        Celluloid::Logger
+      end
+    end
+    def log
+      Celluloid::Logger
+    end
+  end
+end

data/lib/carnivore/runner.rb

@@ -0,0 +1,22 @@
+require 'celluloid'
+require 'carnivore/source'
+require 'carnivore/container'
+
+module Carnivore
+  class << self
+    def configure(&block)
+      mod = Container.new
+      mod.instance_exec(mod, &block)
+      self
+    end
+
+    def start!
+      begin
+        require 'carnivore/supervisor'
+        Supervisor.run
+      rescue Exception => e
+        # Gracefully shut down
+      end
+    end
+  end
+end

data/lib/carnivore/source/http.rb

@@ -0,0 +1,43 @@
+require 'reel'
+require 'carnivore/source'
+
+module Carnivore
+  class Source
+
+    class Http < Source
+
+      attr_reader :args
+
+      def setup(args={})
+        @args = default_args(args)
+      end
+
+      def default_args(args)
+        {
+          :bind => '0.0.0.0',
+          :port => '3000'
+        }.merge(args)
+      end
+
+      def process(*process_args)
+        srv = Reel::Server.supervise(args[:bind], args[:port]) do |con|
+          while(req = con.request)
+            begin
+              msg = format(:request => req, :body => req.body)
+              callbacks.each do |name|
+                c_name = callback_name(name)
+                debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{c_name})>"
+                Celluloid::Actor[c_name].async.call(msg)
+              end
+              con.respond(:ok, 'So long, and thanks for all the fish!')
+            rescue => e
+              con.respond(:bad_request, 'Failed to process request')
+            end
+          end
+        end
+      end
+
+    end
+
+  end
+end

data/lib/carnivore/source/rabbitmq.rb

@@ -0,0 +1,37 @@
+require 'fog'
+require 'carnivore/source'
+
+module Carnivore
+  class Source
+    class RabbitMq < Source
+
+      def setup(args={})
+        @bunny = nil
+        @connection_args = args[:bunny]
+        @queue_name = args[:queue]
+        @exchange_name = args[:exchange]
+        debug "Creating RabbitMq source instance <#{name}>"
+      end
+
+      def connect
+        @bunny = Bunny.new(@connection_args)
+        @bunny.start
+        @channel = @bunny.create_channel
+        @exchange = @channel.topic(@exchange_name)
+        @queue = @channel.queue(@queue_name).bind(@exchange) # TODO: Add topic key
+      end
+
+      def process
+        @queue.subscribe do |info, metadata, payload|
+          msg = format(payload)
+          callbacks.each do |name|
+            c_name = callback_name(name)
+            debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{c_name})>"
+            Celluloid::Actor[c_name].async.call(msg)
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/carnivore/source/sqs.rb

@@ -0,0 +1,76 @@
+require 'fog'
+require 'carnivore/source'
+
+module Carnivore
+  class Source
+    class Sqs < Source
+
+      OUTPUT_REPEAT_EVERY=5
+
+      attr_reader :pause_time
+
+      def setup(args={})
+        @fog = nil
+        @connection_args = args[:fog]
+        @queue = args[:queue_url]
+        @pause_time = args[:pause] || 5
+        @receive_timeout = after(args[:receive_timeout] || 30){ terminate }
+        debug "Creating SQS source instance <#{name}>"
+      end
+
+      def connect
+        @fog = Fog::AWS::SQS.new(@connection_args)
+      end
+
+      def receive(n=1)
+        count = 0
+        m = nil
+        until(m)
+          m = nil
+          @receive_timeout.reset
+          m = @fog.receive_message(@queue, 'MaxNumberOfMessages' => n).body['Message'].first
+          @receive_timeout.reset
+          unless(m)
+            if(count == 0)
+              debug "Source<#{name}> no message received. Sleeping for #{pause_time} seconds."
+            elsif(count % OUTPUT_REPEAT_EVERY == 0)
+              debug "Source<#{name}> last message repeated #{count} times"
+            end
+            sleep(pause_time)
+          end
+          count += 1
+        end
+        pre_process(m)
+      end
+
+      def send(message)
+        @fog.send_message(@queue, message)
+      end
+
+      def confirm(message)
+        @fog.delete_message(@queue, message['ReceiptHandle'])
+      end
+
+      private
+
+      def fog
+        unless(@fog)
+          connect
+        end
+        @fog
+      end
+
+      def pre_process(m)
+        if(m['Body'])
+          begin
+            m['Body'] = JSON.load(m['Body'])
+          rescue JSON::ParserError
+            # well, we did our best
+          end
+        end
+        m
+      end
+
+    end
+  end
+end

data/lib/carnivore/source/test.rb

@@ -0,0 +1,31 @@
+module Carnivore
+  class Source
+    class Test < Source
+
+      RAND_MAX = 99999
+      RAND_DIV = 3
+      RAND_SLEEP = 10
+
+      def setup(args={})
+      end
+
+      def connect(*args)
+        puts 'Test connect called'
+      end
+
+      def receive(*args)
+        if(rand(RAND_MAX) % RAND_DIV == 0)
+          sleep_for = rand(RAND_SLEEP)
+          debug "Test source sleep for: #{sleep_for}"
+          sleep sleep_for
+        end
+        20.times.map{('a'..'z').to_a.shuffle.first}.join
+      end
+
+      def transmit(message)
+        puts "Transmit requested: #{message}"
+      end
+
+    end
+  end
+end

data/lib/carnivore/source.rb

@@ -0,0 +1,151 @@
+require 'carnivore/callback'
+
+module Carnivore
+  class Source < Celluloid::SupervisionGroup
+
+    class SourceContainer
+
+      attr_reader :klass
+      attr_reader :source_hash
+
+      def initialize(class_name, args={})
+        @klass = class_name
+        @source_hash = args || {}
+        @source_hash[:callbacks] = {}
+      end
+
+      def add_callback(name, klass=nil, &block)
+        @source_hash[:callbacks][name] = klass || block
+      end
+    end
+
+    class << self
+
+      def build(args={})
+        [:args, :type].each do |key|
+          unless(args.has_key?(key))
+            raise ArgumentError.new "Missing required parameter `:#{key}`"
+          end
+        end
+        require "carnivore/source/#{args[:type]}"
+        klass = args[:type].to_s.split('_').map(&:capitalize).join
+        klass = Source.const_get(klass)
+        args[:args][:name] ||= Celluloid.uuid
+        inst = SourceContainer.new(klass, args[:args])
+        register(inst)
+        inst
+      end
+
+      def register(inst)
+        @sources ||= []
+        @sources << inst
+        true
+      end
+
+      def sources
+        @sources || []
+      end
+    end
+
+    include Celluloid
+    include Celluloid::Logger
+
+    attr_reader :name
+    attr_reader :callbacks
+    attr_reader :auto_confirm
+    attr_reader :callback_supervisor
+
+    def initialize(args={})
+      @callbacks = []
+      @callback_names = {}
+      @callback_supervisor = Celluloid::SupervisionGroup.run!
+      @name = args[:name] || Celluloid.uuid
+      @auto_confirm = !!args[:auto_confirm]
+      if(args[:callbacks])
+        args[:callbacks].each do |name, block|
+          add_callback(name, block)
+        end
+      end
+      setup(args)
+      connect
+      async.process
+    rescue => e
+      debug "Failed to initialize: #{self} - #{e.class}: #{e}\n#{e.backtrace.join("\n")}"
+      raise
+    end
+
+    def auto_confirm?
+      @auto_confirm
+    end
+
+    def inspect
+      "<#{self.class.name}:#{object_id} @name=#{name} @callbacks=#{Hash[*callbacks.map{|k,v| [k,v.object_id]}.flatten]}>"
+    end
+
+    def setup(args={})
+      debug "<#{self.class}> No custom setup declared"
+    end
+
+    def connect(args={})
+      debug "<#{self.class}> No custom connect declared"
+    end
+
+    def receive(n=1)
+      raise NoMethodError.new('Abstract method not valid for runtime')
+    end
+
+    def transmit(message)
+      raise NoMethodError.new('Abstract method not valid for runtime')
+    end
+
+    def terminate
+      if(@callback_supervisor)
+        @callback_supervisor.actors.map(&:terminate)
+      end
+    end
+
+    def add_callback(name, block_or_class)
+      if(block_or_class.is_a?(Class))
+        debug "Adding callback class (#{block_or_class}) under supervision. Name: #{callback_name(name)}"
+        size = block_or_class.workers || 1
+        @callback_supervisor.pool block_or_class, as: callback_name(name), size: size, args: [name]
+      else
+        debug "Adding custom callback class  from block (#{block_or_class}) under supervision. Name: #{callback_name(name)}"
+        @callback_supervisor.supervise_as callback_name(name), Callback, name, block_or_class
+      end
+      @callbacks.push(name).uniq!
+      self
+    end
+
+    def remove_callback(name)
+      unless(@callbacks.include?(callback_name(name)))
+        raise NameError.new("Failed to locate callback named: #{name}")
+      end
+      actors[callback_name(name)].terminate
+      @callbacks.delete(name)
+      self
+    end
+
+    def callback_name(name)
+      unless(@callback_names[name])
+        @callback_names[name] = [@name, self.object_id, name].join(':').to_sym
+      end
+      @callback_names[name]
+    end
+
+    def format(msg)
+      {:message => msg, :source => self}
+    end
+
+    def process
+      loop do
+        msg = format(receive)
+        @callbacks.each do |name|
+          debug "Dispatching message<#{msg[:message].object_id}> to callback<#{name} (#{callback_name(name)})>"
+          Celluloid::Actor[callback_name(name)].async.call(msg)
+        end
+      end
+    end
+
+  end
+end

data/lib/carnivore/supervisor.rb

@@ -0,0 +1,9 @@
+module Carnivore
+  class Supervisor < Celluloid::SupervisionGroup
+
+    Source.sources.each do |source|
+      supervise source.klass, as: source.source_hash[:name], args: [source.source_hash]
+    end
+
+  end
+end

data/lib/carnivore/utils.rb

@@ -0,0 +1,18 @@
+module Carnivore
+  module Utils
+
+    module Params
+      def symbolize_hash(hash)
+        Hash[*(
+            hash.map do |k,v|
+              [
+                k.gsub(/(?<![A-Z])([A-Z])/, '_\1').sub(/^_/, '').downcase.to_sym,
+                v.is_a?(Hash) ? symbolize_hash(v) : v
+              ]
+            end.flatten
+        )]
+      end
+    end
+
+  end
+end

data/lib/carnivore/version.rb

@@ -0,0 +1,5 @@
+module Carnivore
+  class Version < Gem::Version
+  end
+  VERSION = Version.new('0.1.0')
+end

data/lib/carnivore.rb

@@ -0,0 +1,2 @@
+require 'carnivore/version'
+require 'carnivore/runner'

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: carnivore
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Chris Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: reel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mixlib-config
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Message processing helper
+email: chrisroberts.code@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/carnivore/callback.rb
+- lib/carnivore/supervisor.rb
+- lib/carnivore/source/http.rb
+- lib/carnivore/source/sqs.rb
+- lib/carnivore/source/test.rb
+- lib/carnivore/source/rabbitmq.rb
+- lib/carnivore/source.rb
+- lib/carnivore/version.rb
+- lib/carnivore/runner.rb
+- lib/carnivore/config.rb
+- lib/carnivore/utils.rb
+- lib/carnivore/container.rb
+- lib/carnivore.rb
+- examples/test_http.rb
+- examples/test_class.rb
+- examples/test_block.rb
+- Gemfile
+- README.md
+- CHANGELOG.md
+- Gemfile.lock
+- carnivore.gemspec
+homepage: https://github.com/heavywater/carnivore
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Message processing helper
+test_files: []
+has_rdoc: