qswarm 0.0.21 → 1.0.0

data/lib/qswarm/connection.rb

@@ -0,0 +1,25 @@
+module Qswarm
+  class Connection
+    attr_reader :format
+
+    def initialize(agent, name, args, &block)
+      @agent      = agent
+      @name       = name
+      @args       = args
+      @on_connect = block_given? ? block : false
+
+      @format     = args[:format] || :raw
+    end
+
+    def emit(payload)
+      @agent.emit(@name, :payload => OpenStruct.new(:raw => payload, :format => @format))
+    end
+
+    def sink(args, payload)
+      Qswarm.logger.info ">>> #{payload.raw}"
+    end
+
+    def run
+    end
+  end
+end

data/lib/qswarm/connections/amqp.rb

@@ -0,0 +1,167 @@
+require 'amqp'
+require 'cgi'
+require 'uuid'
+require 'json'
+require 'ostruct'
+
+module Qswarm
+  module Connections
+    class Amqp < Qswarm::Connection
+      include Qswarm::DSL
+
+#    dsl_accessor :name, :host, :port, :user, :pass, :vhost, :exchange_type, :exchange_name, :durable
+      @@connection = {}
+
+      def initialize(agent, name, args, &block)
+        # Set some defaults
+        @host          = 'localhost'
+        @port          = 5672
+        @user          = 'guest'
+        @pass          = 'guest'
+        @vhost         = ''
+        @durable       = true
+        @prefetch      = args[:prefetch] || 0
+
+        decode_uri(args[:uri]) if args[:uri]
+
+        @queues        = {}
+        @channels      = {}
+        @exchange      = nil
+        @instances     = nil
+
+        @queue_args     = { :auto_delete => true, :durable => true, :exclusive => true }.merge! args[:queue_args] || {}
+        @subscribe_args = { :exclusive => false, :ack => false }.merge! args[:subscribe_args] || {}
+        @bind_args      = args[:bind_args] || {}
+        @exchange_type  = args[:exchange_type] || :direct
+        @exchange_name  = args[:exchange_name] || ''
+        @exchange_args  = { :durable => true }.merge! args[:exchange_args] || {}
+        @uuid           = UUID.generate if args[:uniq]
+        @bind           = args[:bind]
+
+        Signal.trap("INT") do
+          @@connection["#{@host}:#{@port}/#{@vhost}"].close do
+            EM.stop { exit }
+          end
+        end
+
+        super
+      end
+
+      def queue(name, routing_key = '', args = nil)
+        @queues["#{name}/#{routing_key}"] ||= begin
+          Qswarm.logger.debug "Binding queue #{name}/#{routing_key}"
+          @queues["#{name}/#{routing_key}"] = channel(name, routing_key).queue(name, args).bind(exchange(channel(name, routing_key)), @bind_args.merge(:routing_key => routing_key))
+        end
+      end
+
+      def exchange(channel = nil)
+        @exchange ||= begin
+          @exchange = AMQP::Exchange.new(channel ||= AMQP::Channel.new(connection, :auto_recovery => true), @exchange_type, @exchange_name, @exchange_args) do |exchange|
+            Qswarm.logger.debug "Declared #{@exchange_type} exchange #{@vhost}/#{@exchange_name}"
+            exchange.on_return do |basic_return, metadata, payload|
+              Qswarm.logger.error "#{payload} was returned! reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}"
+            end
+          end
+        end
+      end
+
+      # ruby-amqp currently limits to 1 consumer per queue (to be fixed in future) so can't pool channels
+      def channel(name, routing_key = '')
+        @channels["#{name}/#{routing_key}"] ||= begin
+          Qswarm.logger.debug "Opening channel for #{name}/#{routing_key}"
+          @channels["#{name}/#{routing_key}"] = AMQP::Channel.new(connection, AMQP::Channel.next_channel_id, :auto_recovery => true, :prefetch => @prefetch) do |c|
+            @channels["#{name}/#{routing_key}"].on_error do |channel, channel_close|
+              Qswarm.logger.error "[channel.close] Reply code = #{channel_close.reply_code}, reply text = #{channel_close.reply_text}"
+            end
+          end
+        end
+      end
+
+      def connection
+        # Pool connections at the class level
+        @@connection["#{@host}:#{@port}/#{@vhost}"] ||= begin
+          Qswarm.logger.debug "Connecting to AMQP broker #{self.to_s}"
+          @@connection["#{@host}:#{@port}/#{@vhost}"] = AMQP.connect(self.to_s, :heartbeat => 30, :on_tcp_connection_failure => Proc.new { |settings|
+              Qswarm.logger.error "AMQP initial connection failure to #{settings[:host]}:#{settings[:port]}/#{settings[:vhost]}"
+              EM.stop
+            }, :on_possible_authentication_failure => Proc.new { |settings|
+              Qswarm.logger.error "AMQP initial authentication failed for #{settings[:host]}:#{settings[:port]}/#{settings[:vhost]}"
+              EM.stop
+            }
+          ) do |c|
+            @@connection["#{@host}:#{@port}/#{@vhost}"].on_recovery do |connection|
+              Qswarm.logger.debug "Recovered from AMQP network failure"
+            end
+            @@connection["#{@host}:#{@port}/#{@vhost}"].on_tcp_connection_loss do |connection|
+              # reconnect in 10 seconds
+              Qswarm.logger.error "AMQP TCP connection lost, reconnecting in 2s"
+              connection.periodically_reconnect(2)
+            end
+            @@connection["#{@host}:#{@port}/#{@vhost}"].on_connection_interruption do |connection|
+              Qswarm.logger.error "AMQP connection interruption"
+            end
+            # Force reconnect on heartbeat loss to cope with our funny firewall issues
+            @@connection["#{@host}:#{@port}/#{@vhost}"].on_skipped_heartbeats do |connection, settings|
+              Qswarm.logger.error "Skipped heartbeats detected"
+            end
+            @@connection["#{@host}:#{@port}/#{@vhost}"].on_error do |connection, connection_close|
+              Qswarm.logger.error "AMQP connection has been closed. Reply code = #{connection_close.reply_code}, reply text = #{connection_close.reply_text}"
+              if connection_close.reply_code == 320
+                Qswarm.logger.error "Set a 30s reconnection timer"
+                # every 30 seconds
+                connection.periodically_reconnect(30)
+              end
+            end
+            Qswarm.logger.debug "Connected to AMQP broker #{self.to_s}"
+          end
+        end
+      end
+
+      def decode_uri(uri)
+        @user, @pass, @host, @port, @vhost = uri.match(/([^:]+):([^@]+)@([^:]+):([^\/]+)\/(.*)/).captures
+      end
+
+      def ack?
+        @subscribe_args[:ack]
+      end
+
+      def to_s
+        "amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape('/' + @vhost)}"
+      end
+
+      def status
+        "AMQP connection #{@name.inspect} at #{@args[:uri]}, bound to #{@args[:bind]}/#{@args[:bind_args]} on #{@args[:exchange_type].inspect} exchange #{@args[:exchange_name]}"
+      end
+
+      def run
+        if !@bind.nil?
+          [*@bind].each do |bind|
+            queue(@agent.name.to_s + '.' +  @name.to_s + @uuid ||= '', bind, @queue_args).subscribe(@subscribe_args) do |metadata, payload|
+              emit metadata, payload
+            end
+          end
+
+          dsl_call(&@on_connect) if @on_connect
+        end
+      end
+
+      def emit(metadata, payload)
+        Qswarm.logger.info "[#{@agent.name.inspect}] :amqp connection #{@name.inspect} bound to #{metadata.routing_key}, received #{payload.inspect}"
+
+        @agent.emit(@name, :payload => OpenStruct.new(:raw => payload, :headers => (metadata.headers.nil? ? {} : Hash[metadata.headers.map{ |k, v| [k.to_sym, v] }]).merge(:routing_key => metadata.routing_key), :format => @format))
+        metadata.ack if ack?
+      end
+
+      def sink(args, payload)
+        [*args[:routing_key]].each do |routing_key|
+          Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Sinking #{payload.raw.inspect} to AMQP routing_key #{routing_key.inspect}"
+          if args[:headers] || payload.headers
+            exchange.publish payload.raw, :routing_key => routing_key, :headers => (args[:headers] ? args[:headers] : payload.headers).merge(:routing_key => routing_key)
+          else
+            exchange.publish payload.raw, :routing_key => routing_key
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qswarm/connections/logger.rb

@@ -0,0 +1,27 @@
+module Qswarm
+  module Connections
+    class Logger < Qswarm::Connection
+      include Qswarm::DSL
+
+      attr_reader :format
+
+      def initialize(agent, name, args, &block)
+        @filename = args[:filename] || '/tmp/qswarm-logger.log'
+
+        super(agent, name, args)
+      end
+
+      def emit(payload)
+      end
+
+      def sink(args, payload)
+        @file.puts payload.raw
+      end
+
+      def run
+        @file = File.open(@filename, 'a')
+        Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Logging to #{@filename}"
+      end
+    end
+  end
+end

data/lib/qswarm/connections/twitter.rb

@@ -0,0 +1,148 @@
+require 'tweetstream'
+require 'twitter'
+require 'json'
+require 'ostruct'
+
+module Qswarm
+  module Connections
+    class Twitter < Qswarm::Connection
+      include Qswarm::DSL
+
+      def initialize(agent, name, args, &block)
+        TweetStream.configure do |config|
+          config.consumer_key = args[:consumer_key]
+          config.consumer_secret = args[:consumer_secret]
+          config.oauth_token = args[:oauth_token]
+          config.oauth_token_secret = args[:oauth_token_secret]
+          config.auth_method = :oauth
+        end
+
+        @rest_client = ::Twitter::Client.new(
+          :consumer_key => args[:consumer_key],
+          :consumer_secret => args[:consumer_secret],
+          :oauth_token => args[:oauth_token],
+          :oauth_token_secret => args[:oauth_token_secret]
+        )
+
+        @track = args[:track]
+        @follow = args[:follow]
+        @list = args[:list]
+
+        super
+      end
+
+      def emit(payload)
+        @agent.emit(@name, :payload => OpenStruct.new(payload))
+      end
+
+      def sink(metadata, payload)
+        Qswarm.logger.info ">>> #{payload}"
+      end
+
+      def status
+        "Connected to tweetstream, tracking #{@track.to_s}, following #{@follow.to_s}"
+      end
+
+      def run
+        begin
+          if @track
+            Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Tracking keywords: " + @track.to_s
+            TweetStream::Client.new.track( @track.values.flatten.reject { |k| /^-/.match(k) } ) do |status|
+              @track.each do |group, list|
+                matches = []
+                list.each do |keyword|
+                  # Text doesn't include any words in the phrase prefixed with -
+                  if keyword.split(' ').select { |k| /^-/.match(k) }.none? { |word| status.text.downcase.include? word[1..-1].downcase }
+                    # Text contains all of the words in the phrase
+                    if keyword.split(' ').reject { |k| /^-/.match(k) }.all? { |word| status.text.downcase.include? word.downcase }
+                      matches << keyword
+                    end
+                  end
+                end
+
+                if !matches.empty?
+                  Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Sending :track/#{group.inspect} #{status.user.screen_name} :: #{status.text} :: #{matches.to_s}"
+                  emit(:raw => status.attrs, :headers => { :type => :track, :group => group, :matches => matches })
+                end
+              end
+#            end.on_limit do |skip_count|
+#              Qswarm.logger.error "[#{@agent.name.inspect} #{@name.inspect}] There were #{skip_count} tweets missed because of rate limiting."
+            end.on_error do |message|
+              Qswarm.logger.error "[#{@agent.name.inspect} #{@name.inspect}] #{message}"
+            end
+          end
+
+          if @follow
+            Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Tracking Users: " + @follow.to_s
+            TweetStream::Client.new.follow( *@follow.values.flatten ) do |status|
+              @follow.each do |group, users|
+                if users.include?(status.user.id)
+                  Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Sending :follow/#{group.inspect} #{status.user.screen_name} :: #{status.text}"
+                  emit(:raw => status.attrs, :headers => { :type => :follow, :group => group, :user_id => status.user.id })
+                end
+              end
+#            end.on_limit do |skip_count|
+#              Qswarm.logger.error "[#{@agent.name.inspect} #{@name.inspect}] There were #{skip_count} tweets missed because of rate limiting."
+            end.on_error do |message|
+              Qswarm.logger.error "[#{@agent.name.inspect} #{@name.inspect}] #{message}"
+            end
+          end
+
+        rescue TweetStream::ReconnectError
+          Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Hit max reconnects, restarting tweetstream in 60 seconds ..."
+          EM.timer(60, run)
+        end
+
+        if @list
+          timer = 30
+          since_id = {}
+
+          Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Tracking List: " + @list.to_s + " every #{timer} seconds"
+
+          @list.each do |group, lists|
+            lists.each do |user, slug|
+              @rest_client.list_timeline(user, slug).each do |status|
+                since_id["#{user}/#{slug}"] = status.attrs[:id] and break
+              end
+            end
+
+            EventMachine::PeriodicTimer.new(timer) do
+              lists.each do |user, slug|
+                begin
+                  @rest_client.list_timeline(user, slug, { :since_id => since_id["#{user}/#{slug}"] }).each do |status|
+                    Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Sending :list/#{slug.inspect} #{status.attrs[:user][:screen_name]} :: #{status.text}"
+                    emit(:raw => status.attrs, :headers => { :type => :list, :group => group, :user_id => user, :slug => slug })
+                    since_id["#{user}/#{slug}"] = status.attrs[:id]
+                  end
+                rescue ::Twitter::Error::ClientError
+                  Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Twitter REST API client error"
+                end
+              end
+            end
+          end
+        end
+
+        dsl_call(&@on_connect) if @on_connect
+      end
+    end
+  end
+end
+
+__END__
+
+            end.on_reconnect do |timeout, retries|
+#            puts "RECONNECT #{retries}"
+            end
+          c.on_error do |message|
+            puts "ERROR #{message}"
+          end
+#  client.on_delete do |status_id, user_id|
+#    puts "DELETE #{status_id}"
+#  end
+
+end
+
+#  c.on_limit do |skip_count|
+#    puts "RATE LIMIT"
+#  end
+

data/lib/qswarm/connections/xmpp.rb

@@ -0,0 +1,92 @@
+require 'blather/client/dsl'
+require 'json'
+
+module Qswarm
+  module Connections
+    class QBlather
+      include Blather::DSL
+
+      Blather.logger.level = Logger::INFO
+
+      def on_connect(block)
+        self.instance_eval(&block)
+      end
+    end
+
+    class Xmpp < Qswarm::Connection
+      include Qswarm::DSL
+
+      def initialize(agent, name, args, &block)
+        @channels = []
+        @connected = false
+        @connection = nil
+        @real_name = args[:real_name] || 'Bot'
+
+        # Use the block for Blather bot DSL
+        super
+      end
+
+      def sink(args, payload)
+        if @connected
+          # Use channel jid argument from write or from connection itself
+          channel = args.nil? || args[:channel].nil? ? @args[:channel] : args[:channel]
+          join channel;
+
+          [*channel].each do |c|
+            Qswarm.logger.info "[#{@agent.name.inspect} #{@name.inspect}] Sinking #{payload.raw.inspect} to XMPP channel #{c.inspect}"
+            @connection.say c, payload.raw, :groupchat
+          end
+        else
+          EventMachine::Timer.new(5,self.sink(args, payload))
+        end
+      end
+
+      def run
+        xmpp_connect @args[:jid], @args[:password], @args[:channel]
+      end
+
+      private
+
+      def join(channel)
+        [*channel].each do |c|
+          next if @channels.include? c
+          Qswarm.logger.debug "Joining XMPP channel #{c}"
+          @connection.join c, @real_name
+          @channels << c
+        end
+      end
+
+      def xmpp_connect(jid, password, channel)
+        Qswarm.logger.debug "Connecting to XMPP server #{jid}"
+
+        s = QBlather.new
+        @connection = s
+
+        s.setup jid, password
+
+        s.when_ready do
+          Qswarm.logger.debug "Connected to XMPP server #{jid}"
+          @connected = true
+          s.on_connect(@on_connect) if @on_connect
+          join channel unless channel.nil?
+          # Hipchat has a 150s inactivity timer
+          EventMachine::PeriodicTimer.new(60) { s << ' ' }
+        end
+
+        s.disconnected do
+          Qswarm.logger.error "Lost XMPP connection to #{jid}, reconnecting..."
+          @connected = false
+          @connection.run
+        end
+
+        EM.defer do
+          s.run
+        end
+      end
+
+      def status
+        "XMPP connected to #{@args[:jid]}, present in channels #{@channel.to_s}"
+      end
+    end
+  end
+end