qswarm 0.0.2 → 0.0.4

data/bin/qswarm

@@ -1,10 +1,16 @@
 #!/usr/bin/env ruby
 
+$stdout.sync = true
+
 require 'qswarm'
 
 usage = "#{$0} <configuration file>"
 abort usage unless config = ARGV.shift
 
+$graylog2_facility = config.split('.')[0]
+$graylog2_host = ARGV.shift || 'localhost'
+
+
 swarm = Qswarm::Swarm.load config
-swarm.log.level = Logger::INFO
+swarm.log.level = Logger::DEBUG
 swarm.run

data/lib/qswarm/agent.rb

@@ -7,11 +7,35 @@ module Qswarm
 
     attr_reader :swarm, :name
 
-    def initialize(swarm, name, &block)
+    def initialize(swarm, name, args, &block)
       @swarm              = swarm
       @name               = name.to_s
       @brokers            = {}
       @listeners          = []
+      @args               = args
+
+      unless args.nil?
+        case args.delete :type
+        when :esper
+          require 'java'
+
+          require 'esper-4.5.0/esper-4.5.0.jar'
+          require 'esper-4.5.0/esper/lib/commons-logging-1.1.1.jar'
+          require 'esper-4.5.0/esper/lib/antlr-runtime-3.2.jar'
+          require 'esper-4.5.0/esper/lib/cglib-nodep-2.2.jar'
+          require 'esper-4.5.0/esper/lib/log4j-1.2.16.jar'
+
+          include_class 'com.espertech.esper.client.EPRuntime'
+          include_class 'com.espertech.esper.client.EPServiceProviderManager'
+          include_class 'com.espertech.esper.client.EPServiceProvider'
+          include_class 'com.espertech.esper.client.EPStatement'
+
+          include_class 'com.espertech.esper.client.UpdateListener'
+          include_class 'com.espertech.esper.client.EventBean'
+          include_class 'org.apache.commons.logging.Log'
+          include_class 'org.apache.commons.logging.LogFactory'
+        end
+      end
 
       self.instance_eval(&block)
     end

data/lib/qswarm/broker.rb

@@ -9,7 +9,7 @@ module Qswarm
     include Qswarm::Loggable
     extend Qswarm::DSL
 
-    dsl_accessor :name, :host, :port, :user, :pass, :vhost, :exchange_type, :exchange_name, :durable, :prefetch
+    dsl_accessor :name, :host, :port, :user, :pass, :vhost, :exchange_type, :exchange_name, :durable
     @@connection = {}
 
     def initialize(name, &block)
@@ -26,6 +26,12 @@ module Qswarm
       @durable       = true
       @prefetch      = nil
 
+      # if block.arity == 1
+      #    yield self
+      #  else
+      #    instance_eval &block
+      #  end
+
       self.instance_eval(&block)
 
       @queues        = {}
@@ -50,6 +56,9 @@ module Qswarm
       @exchange ||= begin
         @exchange = AMQP::Exchange.new(channel ||= AMQP::Channel.new(connection), @exchange_type, @exchange_name, :durable => @durable) do |exchange|
           logger.debug "Declared #{@exchange_type} exchange #{@vhost}/#{@exchange_name}"
+          @exchange.on_return do |basic_return, metadata, payload|
+            logger.error "#{payload} was returned! reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}"
+          end
         end
       end
     end
@@ -58,15 +67,51 @@ module Qswarm
     def channel name, routing_key = ''
       @channels["#{name}/#{routing_key}"] ||= begin
         logger.debug "Opening channel for #{name}/#{routing_key}"
-        @channels["#{name}/#{routing_key}"] = AMQP::Channel.new(connection, :prefetch => @prefetch)
+        @channels["#{name}/#{routing_key}"] = AMQP::Channel.new(connection, AMQP::Channel.next_channel_id, :auto_recovery => true) do |c|
+          @channels["#{name}/#{routing_key}"].on_error do |channel, channel_close|
+            logger.error "[channel.close] Reply code = #{channel_close.reply_code}, reply text = #{channel_close.reply_text}"
+          end
+        end
       end
     end
 
     def connection
       # Pool connections at the class level
       @@connection["#{@host}:#{@port}#{@vhost}"] ||= begin
-        logger.debug "Connecting to AMQP broker at #{@host}:#{@port}#{@vhost}"
-        @@connection["#{@host}:#{@port}#{@vhost}"] = AMQP.connect("amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape(@vhost)}")
+        logger.debug "Connecting to AMQP broker at #{self.to_s}"
+        @@connection["#{@host}:#{@port}#{@vhost}"] = AMQP.connect(self.to_s, :heartbeat => 30, :on_tcp_connection_failure => Proc.new { |settings|
+            logger.error "AMQP initial connection failure to #{settings[:host]}:#{settings[:port]}#{settings[:vhost]}"
+            EM.stop
+          }, :on_possible_authentication_failure => Proc.new { |settings|
+            logger.error "AMQP initial authentication failed for #{settings[:host]}:#{settings[:port]}#{settings[:vhost]}"
+            EM.stop
+          }
+        ) do |c|
+          @@connection["#{@host}:#{@port}#{@vhost}"].on_recovery do |connection|
+            logger.debug "Recovered from AMQP network failure"
+          end
+          @@connection["#{@host}:#{@port}#{@vhost}"].on_tcp_connection_loss do |connection, settings|
+            # reconnect in 10 seconds, without enforcement
+            logger.error "Lost AMQP TCP connection to #{settings[:host]}:#{settings[:port]}#{settings[:vhost]}, reconnecting in 10s"
+            connection.reconnect(false, 10)
+          end
+          # Force reconnect on heartbeat loss to cope with our funny firewall issues
+          @@connection["#{@host}:#{@port}#{@vhost}"].on_skipped_heartbeats do |connection, settings|
+            logger.error "Skipped heartbeats detected, reconnecting in 10s"
+            connection.reconnect(false, 10)
+          end
+          # @@connection["#{@host}:#{@port}#{@vhost}"].on_connection_interruption do |connection|
+          #   logger.error "Connection detected connection interruption"
+          # end
+          @@connection["#{@host}:#{@port}#{@vhost}"].on_error do |connection, connection_close|
+            logger.error "AMQP connection has been closed. Reply code = #{connection_close.reply_code}, reply text = #{connection_close.reply_text}"
+            if connection_close.reply_code == 320
+              logger.error "Set a 30s reconnection timer"
+              # every 30 seconds
+              connection.periodically_reconnect(30)
+            end
+          end
+        end
       end
     end
 
@@ -74,4 +119,4 @@ module Qswarm
       "amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape(@vhost)}"
     end
   end
-end
+end

data/lib/qswarm/listener.rb

@@ -18,10 +18,13 @@ module Qswarm
       @speakers = []
       @sinks    = []
       @format   = :json
+      @instances = nil
 
       @queue_args     = { :auto_delete => true, :durable => true, :exclusive => true }
       @subscribe_args = { :exclusive => false, :ack => false }
 
+      @speaker = args.delete :speaker unless args.nil?
+
       # @subscribe_args.merge! args.delete(:subscribe) unless args.nil?
       @queue_args.merge! args unless args.nil?
 
@@ -34,6 +37,7 @@ module Qswarm
       logger.info "Binding listener #{@name} < #{routing_key}"
     end
 
+    # Not sure about this
     def subscribe(*options)
       Array[*options].each { |o| @subscribe_args[o] = true }
     end
@@ -42,12 +46,22 @@ module Qswarm
       @subscribe_args[:ack]
     end
 
-    def swarm(instances = 1)
+    def swarm(instances = nil)
+      @instances = instances
+    end
+   
+    # Should this be subscribe options?
+    def uniq
       @uuid = '-' + UUID.generate
     end
 
-    def speak(name = nil, &block)
-      @speakers << Qswarm::Speaker.new(self, name, &block)
+    def speak(name, args = nil, &block)
+      if !args.nil?
+        require "qswarm/speakers/#{args[:type].downcase}"
+        @speakers << eval("Qswarm::Speakers::#{args[:type].capitalize}").new(self, name, &block)
+      else
+        @speakers << Qswarm::Speaker.new(self, name, &block)
+      end
       logger.info "Registering speaker: #{name} < #{@name}"
     end
 
@@ -57,9 +71,13 @@ module Qswarm
     end
 
     def run
+      # Any setup that needs to be done
+      @speakers.map { |s| s.run }
+      
       @bind ||= @name
       logger.info "Listening on #{@name} < #{@bind}"
 
+      get_broker.channel(@name, @bind).prefetch(@instances) unless @instances.nil?
       get_broker.queue(@name + @uuid ||= '', @bind, @queue_args).subscribe(@subscribe_args) do |metadata, payload|
         logger.debug "[#{@agent.name}] Received '#{payload.inspect}' on listener #{@name}/#{metadata.routing_key}"
 

data/lib/qswarm/loggable.rb

@@ -1,4 +1,5 @@
 require 'logger'
+require 'gelf'
 
 module Qswarm
   module Loggable
@@ -7,7 +8,8 @@ module Qswarm
     end
 
     def self.logger
-      @logger ||= Logger.new(STDOUT)
+      # @logger ||= Logger.new(STDOUT)
+      @logger ||= GELF::Logger.new($graylog2_host, 12201, 'WAN', { :facility => $graylog2_facility })
     end
   end
-end
+end

data/lib/qswarm/speaker.rb

@@ -27,7 +27,7 @@ module Qswarm
         @heard = payload
       end
 
-      self.instance_exec(&@block)
+      self.instance_eval(&@block)
     rescue JSON::ParserError
       error = "JSON::ParserError on #{payload.inspect}"
       logger.error error
@@ -47,6 +47,9 @@ module Qswarm
     def get_broker(name)
       @listener.get_broker(name)
     end
+    
+    def run
+    end
 
     private
 

data/lib/qswarm/speakers/http.rb

@@ -0,0 +1,48 @@
+require 'uri'
+require 'em-http-request'
+
+module Qswarm
+  module Speakers
+    class Http < Qswarm::Speaker
+      @@connections = {}
+
+      def initialize(listener, name, &block)
+        @uri = URI.parse(name)
+        @uri.host = 'localhost' if @uri.host.nil?
+        super
+      end
+
+      def inject(format = :text, msg)
+        publish format, msg
+      end
+
+      def run
+        connect
+      end
+
+      private
+
+      def connect
+      end
+
+      def publish(format, msg)
+        logger.debug "Sending '#{msg}' to #{@name}"
+        
+        case format
+        when :get
+          if msg.is_a? Hash
+            http = EventMachine::HttpRequest.new(@uri).get :query => msg
+          else
+            http = EventMachine::HttpRequest.new(URI.join(@uri.to_s, msg)).get
+          end
+          http.errback do |err|
+            logger.error "Error sending #{msg} to #{@name}: #{err}"
+          end
+          http.callback do 
+            logger.debug "#{http.response_header.status} #{http.response}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qswarm/speakers/irc.rb

@@ -0,0 +1,45 @@
+require 'cinch'
+
+module Qswarm
+  module Speakers
+    class Irc < Qswarm::Speaker
+      @@irc_servers = {}
+        
+      def inject(format = :text, msg)
+        logger.debug "[#{@agent.name}] Sending '#{msg}' to channel #{@name}"
+        publish format, @name, msg
+      end
+
+      def run
+        @name.match(/([^#]*)(#.+)/) { irc_connect $1.empty? ? 'localhost' : $1, $2 }
+      end
+      
+      private
+      
+      def irc_connect(irc_server, channel)
+        if bot = @@irc_servers[irc_server]
+          logger.debug "Joining channel #{channel}"
+          @@irc_servers[irc_server].config.channels << channel
+        else
+          logger.debug "Connecting to IRC server #{irc_server} channel #{channel}"
+          @@irc_servers[irc_server]               = Cinch::Bot.new
+          @@irc_servers[irc_server].config.server = irc_server
+          @@irc_servers[irc_server].config.nick   = @agent.name
+          @@irc_servers[irc_server].config.channels << channel
+          
+          EM.defer do
+            @@irc_servers[irc_server].start
+          end
+        end
+      end
+
+      def publish(format, name, msg)
+        @name.match(/([^#]*)(#.+)/) do
+          @@irc_servers[$1.empty? ? 'localhost' : $1].channel_manager.find_ensured($2).andand.send(
+            format == :json ? JSON.generate(msg) : msg
+          )
+        end  
+      end
+    end
+  end
+end

data/lib/qswarm/speakers/mysql.rb

@@ -0,0 +1,39 @@
+# require 'mysql2'
+
+module Qswarm
+  module Speakers
+    class Mysql < Qswarm::Speaker
+      @@db_servers = {}
+
+      def inject(format = :text, msg)
+        logger.debug "[#{@agent.name}] Sending '#{msg}' to channel #{@name}"
+        publish format, @name, msg
+      end
+
+      def run
+        @name.match(/([^#]*)(#.+)/) { db_connect $1.empty? ? 'localhost' : $1, $2 }
+      end
+
+      private
+
+      def db_connect(db_server, database)
+        if connection = @@db_servers[db_server]
+          logger.debug "Connecting to database #{database}"
+        else
+          logger.debug "Connecting to DB server #{db_server} database #{database}"
+          @@db_servers[db_server]               = 'foo'
+        end
+      end
+
+      def publish(format, name, msg)
+        @name.match(/([^#]*)(#.+)/) do
+          logger.debug "#{$1} #{$2} #{msg}"
+          # @@db_servers[$1.empty? ? 'localhost' : $1].channel_manager.find_ensured($2).andand.send(
+          #    format == :json ? JSON.generate(msg) : msg
+          #  )
+          #         end  
+        end
+      end
+    end
+  end
+end

data/lib/qswarm/speakers/nc.rb

@@ -0,0 +1,41 @@
+module Qswarm
+  module Speakers
+    class Nc < Qswarm::Speaker
+      @@connections = {}
+
+      def inject(format = :text, msg)
+        publish format, msg
+      end
+
+      def run
+        @name.match(/([^#]*):(\d+)/) do
+          @host = $1.empty? ? 'localhost' : $1
+          @port = $2
+          @server = @host + ':' + @port
+          # @service = $3
+        end
+        connect
+      end
+
+      private
+
+      def connect
+        if connection = @@connections[@server]
+          logger.debug "Connecting to service #{@service}"
+        else
+          logger.debug "Connecting to host #{@server} service #{@service}"
+          @@connections[@server] = EventMachine::connect @host, @port do |connection|
+            def connection.receive_data(data)
+              puts "Received #{data} from #{@name}"
+            end
+          end
+        end
+      end
+
+      def publish(format, msg)
+        logger.debug "Sending '#{msg}' to #{@name}"
+        @@connections[@server].send_data( (format == :json ? JSON.generate(msg) : msg) + "\n")
+      end
+    end
+  end
+end

data/lib/qswarm/swarm.rb

@@ -24,9 +24,9 @@ module Qswarm
       logger
     end
 
-    def agent(name, &block)
+    def agent(name, args = nil, &block)
       logger.info "Registering agent: #{name}"
-      @agents << Qswarm::Agent.new(self, name, &block)
+      @agents << Qswarm::Agent.new(self, name, args, &block)
     end
 
     def broker(name, &block)

data/lib/qswarm/version.rb

@@ -1,3 +1,3 @@
 module Qswarm
-  VERSION = "0.0.2"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: qswarm
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-29 00:00:00.000000000 +00:00
-default_executable: 
+date: 2012-03-14 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &2157460660 !ruby/object:Gem::Requirement
+  requirement: &70191652850140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -22,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157460660
+  version_requirements: *70191652850140
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &2157460240 !ruby/object:Gem::Requirement
+  requirement: &70191652849720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -33,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157460240
+  version_requirements: *70191652849720
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &2157459820 !ruby/object:Gem::Requirement
+  requirement: &70191652849300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -44,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157459820
+  version_requirements: *70191652849300
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &2157459400 !ruby/object:Gem::Requirement
+  requirement: &70191652848880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -55,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157459400
+  version_requirements: *70191652848880
 - !ruby/object:Gem::Dependency
   name: andand
-  requirement: &2157458980 !ruby/object:Gem::Requirement
+  requirement: &70191652848460 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -66,7 +65,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *2157458980
+  version_requirements: *70191652848460
 description: Framework for writing distributed agents hanging off an AMQP message
   bus
 email:
@@ -88,10 +87,13 @@ files:
 - lib/qswarm/listener.rb
 - lib/qswarm/loggable.rb
 - lib/qswarm/speaker.rb
+- lib/qswarm/speakers/http.rb
+- lib/qswarm/speakers/irc.rb
+- lib/qswarm/speakers/mysql.rb
+- lib/qswarm/speakers/nc.rb
 - lib/qswarm/swarm.rb
 - lib/qswarm/version.rb
 - qswarm.gemspec
-has_rdoc: true
 homepage: http://github.com/ennui2342/qswarm
 licenses: []
 post_install_message: 
@@ -112,7 +114,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: qswarm
-rubygems_version: 1.6.2
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Distributed queue based agents in Ruby