qswarm 0.0.21 → 1.0.0

data/lib/qswarm/dsl.rb

@@ -1,17 +1,61 @@
 module Qswarm
   module DSL
-    def dsl_accessor(*symbols)
-      symbols.each { |sym|
-        class_eval %{
-          def #{sym}(*val)
-            if val.empty?
-              @#{sym}
-            else
-              @#{sym} = val.size == 1 ? val[0] : val
+    module Config
+      @@caller = nil
+
+      def self.caller
+        @@caller
+      end
+
+      def self.caller=(caller)
+        @@caller = caller
+      end
+
+      class Swarm; extend Qswarm::DSL::Config; end
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def dsl_load(config)
+      dsl_call(File.read(config))
+    end
+
+    def dsl_call(string = nil, &block)
+      parent = Config.caller
+      Config.caller = self
+      if string.nil?
+        res = Config::Swarm.module_eval(&block)
+      else
+        res = Config::Swarm.module_eval(string)
+      end
+      Config.caller = parent
+      res
+    end
+
+    module ClassMethods
+      def dsl_accessor(*symbols)
+        symbols.each do |sym|
+          Qswarm::DSL::Config.module_eval %{
+            def #{sym}
+              @@#{sym}
+            end
+
+            def #{sym}=(*val)
+              @@#{sym} = val.size == 1 ? val[0] : val
             end
-          end
-        }
-      }
+          }
+        end
+      end
+
+      def dsl(*symbols)
+        symbols.each do |sym|
+          Qswarm::DSL::Config.module_eval "def #{sym}(*args, &block) @@caller.send(#{sym.inspect}, *args, &block); end"
+          #Qswarm::DSL::Config.module_eval "def #{sym}(name, args = nil, &block) @@caller.send(#{sym.inspect}, name, args, &block); end"
+          #Qswarm::DSL::Config.module_eval { define_method(sym, -> (name, args = nil, &block) { @@caller.send(sym.inspect, name, args, &block) } ) }
+        end
+      end
     end
   end
-end
+end

data/lib/qswarm/swarm.rb

@@ -1,43 +1,24 @@
 require 'andand'
 require 'eventmachine'
 
-require 'qswarm/agent'
-require 'qswarm/broker'
-
 module Qswarm
   class Swarm
-    include Qswarm::Loggable
+    include Qswarm::DSL
 
-    def self.load(config)
-      dsl = new
-      dsl.instance_eval(File.read(config), config)
-      dsl
-    end
+    dsl :agent
 
-    def initialize
+    def initialize(config)
       @agents = []
       $fqdn = Socket.gethostbyname(Socket.gethostname).first
-      @brokers = {}
-    end
 
-    def log
-      logger
+      dsl_load(config)
     end
 
     def agent(name, args = nil, &block)
-      logger.info "Registering agent: #{name}"
+      Qswarm.logger.info "Registering agent: #{name.inspect}"
       @agents << Qswarm::Agent.new(self, name, args, &block)
     end
 
-    def broker(name, &block)
-      logger.info "Registering broker: #{name}"
-      @brokers[name] = Qswarm::Broker.new(name, &block)
-    end
-
-    def get_broker(name)
-      @brokers[name]
-    end
-
     def run
       EventMachine.run do
         @agents.map { |a| a.run }

data/lib/qswarm/version.rb

@@ -1,3 +1,3 @@
 module Qswarm
-  VERSION = "0.0.21"
+  VERSION = "1.0.0"
 end

data/qswarm.gemspec

@@ -8,8 +8,8 @@ Gem::Specification.new do |s|
   s.authors     = ["Mark Cheverton"]
   s.email       = ["mark.cheverton@ecafe.org"]
   s.homepage    = "http://github.com/ennui2342/qswarm"
-  s.summary     = %q{Distributed queue based agents in Ruby}
-  s.description = %q{Framework for writing distributed agents hanging off an AMQP message bus}
+  s.summary     = %q{Streaming event processing DSL for Ruby}
+  s.description = %q{Defines a DSL to allow stream processing from various sources for output to various sinks}
 
   s.rubyforge_project = "qswarm"
 
@@ -27,4 +27,8 @@ Gem::Specification.new do |s|
   s.add_dependency 'uuid'
   s.add_dependency 'json'
   s.add_dependency 'andand'
+  s.add_dependency 'nokogiri'
+  s.add_dependency 'tweetstream'
+  s.add_dependency 'twitter'
+  s.add_dependency 'trollop'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: qswarm
 version: !ruby/object:Gem::Version
-  version: 0.0.21
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-21 00:00:00.000000000Z
+date: 2013-12-17 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70093220631160 !ruby/object:Gem::Requirement
+  requirement: &70217708776120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70093220631160
+  version_requirements: *70217708776120
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &70093220630740 !ruby/object:Gem::Requirement
+  requirement: &70217708775700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70093220630740
+  version_requirements: *70217708775700
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &70093220630320 !ruby/object:Gem::Requirement
+  requirement: &70217708775280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70093220630320
+  version_requirements: *70217708775280
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70093220629900 !ruby/object:Gem::Requirement
+  requirement: &70217708774860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70093220629900
+  version_requirements: *70217708774860
 - !ruby/object:Gem::Dependency
   name: andand
-  requirement: &70093220629480 !ruby/object:Gem::Requirement
+  requirement: &70217708774440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,9 +65,53 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70093220629480
-description: Framework for writing distributed agents hanging off an AMQP message
-  bus
+  version_requirements: *70217708774440
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70217708774020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70217708774020
+- !ruby/object:Gem::Dependency
+  name: tweetstream
+  requirement: &70217708773600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70217708773600
+- !ruby/object:Gem::Dependency
+  name: twitter
+  requirement: &70217708773180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70217708773180
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: &70217708772760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70217708772760
+description: Defines a DSL to allow stream processing from various sources for output
+  to various sinks
 email:
 - mark.cheverton@ecafe.org
 executables:
@@ -82,15 +126,12 @@ files:
 - bin/qswarm
 - lib/qswarm.rb
 - lib/qswarm/agent.rb
-- lib/qswarm/broker.rb
+- lib/qswarm/connection.rb
+- lib/qswarm/connections/amqp.rb
+- lib/qswarm/connections/logger.rb
+- lib/qswarm/connections/twitter.rb
+- lib/qswarm/connections/xmpp.rb
 - lib/qswarm/dsl.rb
-- lib/qswarm/listener.rb
-- lib/qswarm/loggable.rb
-- lib/qswarm/speaker.rb
-- lib/qswarm/speakers/http.rb
-- lib/qswarm/speakers/irc.rb
-- lib/qswarm/speakers/mysql.rb
-- lib/qswarm/speakers/nc.rb
 - lib/qswarm/swarm.rb
 - lib/qswarm/version.rb
 - qswarm.gemspec
@@ -117,5 +158,5 @@ rubyforge_project: qswarm
 rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
-summary: Distributed queue based agents in Ruby
+summary: Streaming event processing DSL for Ruby
 test_files: []

data/lib/qswarm/broker.rb

@@ -1,118 +0,0 @@
-require 'eventmachine'
-require 'amqp'
-require 'cgi'
-
-require 'qswarm/dsl'
-
-module Qswarm
-  class Broker
-    include Qswarm::Loggable
-    extend Qswarm::DSL
-
-    dsl_accessor :name, :host, :port, :user, :pass, :vhost, :exchange_type, :exchange_name, :durable
-    @@connection = {}
-
-    def initialize(name, &block)
-      @name          = name
-
-      # Set some defaults
-      @host          = 'localhost'
-      @port          = 5672
-      @user          = 'guest'
-      @pass          = 'guest'
-      @vhost         = ''
-      @exchange_type = :direct
-      @exchange_name = ''
-      @durable       = true
-      @prefetch      = nil
-
-      # if block.arity == 1
-      #    yield self
-      #  else
-      #    instance_eval &block
-      #  end
-
-      self.instance_eval(&block)
-
-      @queues        = {}
-      @channels      = {}
-      @exchange      = nil
-
-      Signal.trap("INT") do
-        @@connection["#{@host}:#{@port}#{@vhost}"].close do
-          EM.stop { exit }
-        end
-      end
-    end
-
-    def queue name, routing_key = '', args = nil
-      @queues["#{name}/#{routing_key}"] ||= begin
-        logger.debug "Binding queue #{name}/#{routing_key}"
-        @queues["#{name}/#{routing_key}"] = channel(name, routing_key).queue(name, args).bind(exchange(channel(name, routing_key)), :routing_key => routing_key)
-      end
-    end
-
-    def exchange(channel = nil)
-      @exchange ||= begin
-        @exchange = AMQP::Exchange.new(channel ||= AMQP::Channel.new(connection, :auto_recovery => true), @exchange_type, @exchange_name, :durable => @durable) do |exchange|
-          logger.debug "Declared #{@exchange_type} exchange #{@vhost}/#{@exchange_name}"
-          @exchange.on_return do |basic_return, metadata, payload|
-            logger.error "#{payload} was returned! reply_code = #{basic_return.reply_code}, reply_text = #{basic_return.reply_text}"
-          end
-        end
-      end
-    end
-
-    # ruby-amqp currently limits to 1 consumer per queue (to be fixed in future) so can't pool channels
-    def channel name, routing_key = ''
-      @channels["#{name}/#{routing_key}"] ||= begin
-        logger.debug "Opening channel for #{name}/#{routing_key}"
-        @channels["#{name}/#{routing_key}"] = AMQP::Channel.new(connection, AMQP::Channel.next_channel_id, :auto_recovery => true) do |c|
-          @channels["#{name}/#{routing_key}"].on_error do |channel, channel_close|
-            logger.error "[channel.close] Reply code = #{channel_close.reply_code}, reply text = #{channel_close.reply_text}"
-          end
-        end
-      end
-    end
-
-    def connection
-      # Pool connections at the class level
-      @@connection["#{@host}:#{@port}#{@vhost}"] ||= begin
-        logger.debug "Connecting to AMQP broker at #{self.to_s}"
-        @@connection["#{@host}:#{@port}#{@vhost}"] = AMQP.connect(self.to_s, :heartbeat => 30, :on_tcp_connection_failure => Proc.new { |settings|
-            logger.error "AMQP initial connection failure to #{settings[:host]}:#{settings[:port]}#{settings[:vhost]}"
-            EM.stop
-          }, :on_possible_authentication_failure => Proc.new { |settings|
-            logger.error "AMQP initial authentication failed for #{settings[:host]}:#{settings[:port]}#{settings[:vhost]}"
-            EM.stop
-          }
-        ) do |c|
-          @@connection["#{@host}:#{@port}#{@vhost}"].on_recovery do |connection|
-            logger.debug "Recovered from AMQP network failure"
-          end
-          @@connection["#{@host}:#{@port}#{@vhost}"].on_connection_interruption do |connection|
-            # reconnect in 10 seconds
-            logger.error "AMQP connection interruption, reconnecting in 10s"
-            connection.reconnect(false, 10)
-          end
-          # Force reconnect on heartbeat loss to cope with our funny firewall issues
-          @@connection["#{@host}:#{@port}#{@vhost}"].on_skipped_heartbeats do |connection, settings|
-            logger.error "Skipped heartbeats detected"
-          end
-          @@connection["#{@host}:#{@port}#{@vhost}"].on_error do |connection, connection_close|
-            logger.error "AMQP connection has been closed. Reply code = #{connection_close.reply_code}, reply text = #{connection_close.reply_text}"
-            if connection_close.reply_code == 320
-              logger.error "Set a 30s reconnection timer"
-              # every 30 seconds
-              connection.periodically_reconnect(30)
-            end
-          end
-        end
-      end
-    end
-
-    def to_s
-      "amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape(@vhost)}"
-    end
-  end
-end

data/lib/qswarm/listener.rb

@@ -1,99 +0,0 @@
-require 'uuid'
-require 'eventmachine'
-
-require 'qswarm/dsl'
-require 'qswarm/speaker'
-
-module Qswarm
-  class Listener
-    include Qswarm::Loggable
-    extend Qswarm::DSL
-
-    dsl_accessor :name, :broker, :format
-    attr_reader :agent
-
-    def initialize(agent, name, args, &block)
-      @agent    = agent
-      @name     = name.to_s
-      @speakers = []
-      @sinks    = []
-      @format   = :json
-      @instances = nil
-
-      @queue_args     = { :auto_delete => true, :durable => true, :exclusive => true }
-      @subscribe_args = { :exclusive => false, :ack => false }
-
-      @speaker = args.delete :speaker unless args.nil?
-
-      # @subscribe_args.merge! args.delete(:subscribe) unless args.nil?
-      @queue_args.merge! args unless args.nil?
-
-      self.instance_eval(&block)
-    end
-
-    def bind(routing_key, options = nil)
-      @bind = routing_key
-      @queue_args.merge! options unless options.nil?
-      logger.info "Binding listener #{@name} < #{routing_key}"
-    end
-
-    # Not sure about this
-    def subscribe(*options)
-      Array[*options].each { |o| @subscribe_args[o] = true }
-    end
-
-    def ack?
-      @subscribe_args[:ack]
-    end
-
-    def swarm(instances = nil)
-      @instances = instances
-    end
-   
-    # Should this be subscribe options?
-    def uniq
-      @uuid = '-' + UUID.generate
-    end
-
-    def speak(name, args = nil, &block)
-      if !args.nil? && !args[:type].nil?
-        require "qswarm/speakers/#{args[:type].downcase}"
-        @speakers << eval("Qswarm::Speakers::#{args[:type].capitalize}").new(self, name, args, &block)
-      else
-        @speakers << Qswarm::Speaker.new(self, name, args, &block)
-      end
-      logger.info "Registering speaker: #{name} < #{@name}"
-    end
-
-    def get_broker(name = nil)
-      name ||= @broker
-      @agent.get_broker(name)
-    end
-
-    def run
-      # Any setup that needs to be done
-      @speakers.map { |s| s.run }
-      
-      @bind ||= @name
-      logger.info "Listening on #{@name} < #{@bind}"
-
-      get_broker.channel(@name, @bind).prefetch(@instances) unless @instances.nil?
-      get_broker.queue(@name + @uuid ||= '', @bind, @queue_args).subscribe(@subscribe_args) do |metadata, payload|
-        logger.debug "[#{@agent.name}] Received '#{payload.inspect}' on listener #{@name}/#{metadata.routing_key}"
-
-        running = @speakers.map { |s| s.object_id }
-        callback = proc do |speaker|
-          running.delete speaker.object_id
-          metadata.ack if ack? && running.empty?
-        end
-
-        @speakers.map do |speaker|
-          EM.defer nil, callback do
-            speaker.parse(metadata, payload)
-            speaker
-          end
-        end
-      end
-    end
-  end
-end