qswarm 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in qswarm.gemspec
+gemspec

data/README.md

@@ -0,0 +1 @@
+Work in progress

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/qswarm

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+require 'qswarm'
+
+usage = "#{$0} <configuration file>"
+abort usage unless config = ARGV.shift
+
+swarm = Qswarm::Swarm.load config
+swarm.log.level = Logger::INFO
+swarm.run

data/lib/qswarm/agent.rb

@@ -0,0 +1,41 @@
+require 'qswarm/broker'
+require 'qswarm/listener'
+
+module Qswarm
+  class Agent
+    include Qswarm::Loggable
+
+    attr_reader :swarm, :name
+
+    def initialize(swarm, name, &block)
+      @swarm              = swarm
+      @name               = name.to_s
+      @brokers            = {}
+      @listeners          = []
+
+      self.instance_eval(&block)
+    end
+
+    def listen(name, args = nil, &block)
+      logger.info "Registering listener: #{name}"
+      @listeners << Qswarm::Listener.new(self, name, args, &block)
+    end
+
+    def broker(name, &block)
+      logger.info "Registering broker: #{name}"
+      @brokers[name] = Qswarm::Broker.new(name, &block)
+    end
+
+    def get_broker(name)
+      @brokers[name] || @swarm.get_broker(name)
+    end
+
+    def bind
+      logger.info "Binding to exchange"
+    end
+
+    def run
+      @listeners.map { |l| l.run }
+    end
+  end
+end

data/lib/qswarm/broker.rb

@@ -0,0 +1,77 @@
+require 'eventmachine'
+require 'amqp'
+require 'cgi'
+
+require 'qswarm/dsl'
+
+module Qswarm
+  class Broker
+    include Qswarm::Loggable
+    extend Qswarm::DSL
+
+    dsl_accessor :name, :host, :port, :user, :pass, :vhost, :exchange_type, :exchange_name, :durable, :prefetch
+    @@connection = {}
+
+    def initialize(name, &block)
+      @name          = name
+
+      # Set some defaults
+      @host          = 'localhost'
+      @port          = 5672
+      @user          = 'guest'
+      @pass          = 'guest'
+      @vhost         = ''
+      @exchange_type = :direct
+      @exchange_name = ''
+      @durable       = true
+      @prefetch      = nil
+
+      self.instance_eval(&block)
+
+      @queues        = {}
+      @channels      = {}
+      @exchange      = nil
+
+      Signal.trap("INT") do
+        @@connection["#{@host}:#{@port}#{@vhost}"].close do
+          EM.stop { exit }
+        end
+      end
+    end
+
+    def queue name, routing_key = '', args = nil
+      @queues["#{name}/#{routing_key}"] ||= begin
+        logger.debug "Binding queue #{name}/#{routing_key}"
+        @queues["#{name}/#{routing_key}"] = channel(name, routing_key).queue(name, args).bind(exchange(channel(name, routing_key)), :routing_key => routing_key)
+      end
+    end
+
+    def exchange(channel = nil)
+      @exchange ||= begin
+        @exchange = AMQP::Exchange.new(channel ||= AMQP::Channel.new(connection), @exchange_type, @exchange_name, :durable => @durable) do |exchange|
+          logger.debug "Declared #{@exchange_type} exchange #{@vhost}/#{@exchange_name}"
+        end
+      end
+    end
+
+    # ruby-amqp currently limits to 1 consumer per queue (to be fixed in future) so can't pool channels
+    def channel name, routing_key = ''
+      @channels["#{name}/#{routing_key}"] ||= begin
+        logger.debug "Opening channel for #{name}/#{routing_key}"
+        @channels["#{name}/#{routing_key}"] = AMQP::Channel.new(connection, :prefetch => @prefetch)
+      end
+    end
+
+    def connection
+      # Pool connections at the class level
+      @@connection["#{@host}:#{@port}#{@vhost}"] ||= begin
+        logger.debug "Connecting to AMQP broker at #{@host}:#{@port}#{@vhost}"
+        @@connection["#{@host}:#{@port}#{@vhost}"] = AMQP.connect("amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape(@vhost)}")
+      end
+    end
+
+    def to_s
+      "amqp://#{@user}:#{@pass}@#{@host}:#{@port}/#{CGI.escape(@vhost)}"
+    end
+  end
+end

data/lib/qswarm/dsl.rb

@@ -0,0 +1,17 @@
+module Qswarm
+  module DSL
+    def dsl_accessor(*symbols)
+      symbols.each { |sym|
+        class_eval %{
+          def #{sym}(*val)
+            if val.empty?
+              @#{sym}
+            else
+              @#{sym} = val.size == 1 ? val[0] : val
+            end
+          end
+        }
+      }
+    end
+  end
+end

data/lib/qswarm/listener.rb

@@ -0,0 +1,81 @@
+require 'uuid'
+require 'eventmachine'
+
+require 'qswarm/dsl'
+require 'qswarm/speaker'
+
+module Qswarm
+  class Listener
+    include Qswarm::Loggable
+    extend Qswarm::DSL
+
+    dsl_accessor :name, :broker, :format
+    attr_reader :agent
+
+    def initialize(agent, name, args, &block)
+      @agent    = agent
+      @name     = name.to_s
+      @speakers = []
+      @sinks    = []
+      @format   = :json
+
+      @queue_args     = { :auto_delete => true, :durable => true, :exclusive => true }
+      @subscribe_args = { :exclusive => false, :ack => false }
+
+      # @subscribe_args.merge! args.delete(:subscribe) unless args.nil?
+      @queue_args.merge! args unless args.nil?
+
+      self.instance_eval(&block)
+    end
+
+    def bind(routing_key, options = nil)
+      @bind = routing_key
+      @queue_args.merge! options unless options.nil?
+      logger.info "Binding listener #{@name} < #{routing_key}"
+    end
+
+    def subscribe(*options)
+      Array[*options].each { |o| @subscribe_args[o] = true }
+    end
+
+    def ack?
+      @subscribe_args[:ack]
+    end
+
+    def swarm(instances = 1)
+      @uuid = '-' + UUID.generate
+    end
+
+    def speak(name = nil, &block)
+      @speakers << Qswarm::Speaker.new(self, name, &block)
+      logger.info "Registering speaker: #{name} < #{@name}"
+    end
+
+    def get_broker(name = nil)
+      name ||= @broker
+      @agent.get_broker(name)
+    end
+
+    def run
+      @bind ||= @name
+      logger.info "Listening on #{@name} < #{@bind}"
+
+      get_broker.queue(@name + @uuid ||= '', @bind, @queue_args).subscribe(@subscribe_args) do |metadata, payload|
+        logger.debug "[#{@agent.name}] Received '#{payload.inspect}' on listener #{@name}/#{metadata.routing_key}"
+
+        running = @speakers.map { |s| s.object_id }
+        callback = proc do |speaker|
+          running.delete speaker.object_id
+          metadata.ack if ack? && running.empty?
+        end
+
+        @speakers.map do |speaker|
+          EM.defer nil, callback do
+            speaker.parse(metadata, payload)
+            speaker
+          end
+        end
+      end
+    end
+  end
+end

data/lib/qswarm/loggable.rb

@@ -0,0 +1,13 @@
+require 'logger'
+
+module Qswarm
+  module Loggable
+    def logger
+      Loggable.logger
+    end
+
+    def self.logger
+      @logger ||= Logger.new(STDOUT)
+    end
+  end
+end

data/lib/qswarm/speaker.rb

@@ -0,0 +1,62 @@
+require 'json'
+
+require 'qswarm/dsl'
+
+module Qswarm
+  class Speaker
+    include Qswarm::Loggable
+    extend Qswarm::DSL
+
+    dsl_accessor :broker
+    attr_reader :agent, :metadata, :heard, :name
+
+    def initialize(listener, name, &block)
+      @listener = listener
+      @agent    = @listener.agent
+      @name     = name.to_s unless name.nil?
+      @block    = block
+
+    end
+
+    def parse(metadata, payload)
+      @metadata = metadata
+      case @listener.format
+      when :json
+        @heard = JSON.parse(payload)
+      else
+        @heard = payload
+      end
+
+      self.instance_exec(&@block)
+    rescue JSON::ParserError
+      error = "[#{@agent.name}@#{$fqdn}] JSON::ParserError on #{payload.inspect}"
+      logger.error error
+      publish :errors, :text, 'nomad.errors', error
+    end
+
+    def log(msg)
+      logger.info "[#{@agent.name}] #{msg}"
+    end
+
+    def inject(format = :text, msg)
+      logger.debug "[#{@agent.name}] Sending '#{msg}' to broker #{get_broker(@broker).name}/#{@name}"
+      publish @broker, format, @name, msg
+      log msg if format == :text
+    end
+
+    def get_broker(name)
+      @listener.get_broker(name)
+    end
+
+    private
+
+    def publish(broker_name, format, routing_key, msg)
+      case format
+      when :json
+        get_broker(broker_name).exchange.publish JSON.generate(msg), :routing_key => routing_key
+      when :text
+        get_broker(broker_name).exchange.publish msg, :routing_key => routing_key
+      end    
+    end
+  end
+end

data/lib/qswarm/swarm.rb

@@ -0,0 +1,51 @@
+require 'andand'
+require 'eventmachine'
+
+require 'qswarm/agent'
+require 'qswarm/broker'
+require 'qswarm/dsl'
+
+module Qswarm
+  class Swarm
+    include Qswarm::Loggable
+    extend Qswarm::DSL
+
+    dsl_accessor :errors_queue
+
+    def self.load(config)
+      dsl = new
+      dsl.instance_eval(File.read(config), config)
+      dsl
+    end
+
+    def initialize
+      @agents = []
+      $fqdn = Socket.gethostbyname(Socket.gethostname).first
+      @brokers = {}
+    end
+
+    def log
+      logger
+    end
+
+    def agent(name, &block)
+      logger.info "Registering agent: #{name}"
+      @agents << Qswarm::Agent.new(self, name, &block)
+    end
+
+    def broker(name, &block)
+      logger.info "Registering broker: #{name}"
+      @brokers[name] = Qswarm::Broker.new(name, &block)
+    end
+
+    def get_broker(name)
+      @brokers[name]
+    end
+
+    def run
+      EventMachine.run do
+        @agents.map { |a| a.run }
+      end
+    end
+  end
+end

data/lib/qswarm/version.rb

@@ -0,0 +1,3 @@
+module Qswarm
+  VERSION = "0.0.1"
+end

data/lib/qswarm.rb

@@ -0,0 +1,3 @@
+require "qswarm/version"
+require 'qswarm/loggable'
+require 'qswarm/swarm'

data/qswarm.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "qswarm/version"
+
+Gem::Specification.new do |s|
+  s.name        = "qswarm"
+  s.version     = Qswarm::VERSION
+  s.authors     = ["Mark Cheverton"]
+  s.email       = ["mark.cheverton@ecafe.org"]
+  s.homepage    = "http://github.com/ennui2342/qswarm"
+  s.summary     = %q{Distributed queue based agents in Ruby}
+  s.description = %q{Framework for writing distributed agents hanging off an AMQP message bus}
+
+  s.rubyforge_project = "qswarm"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  # specify any dependencies here; for example:
+  # s.add_development_dependency "rspec"
+  # s.add_runtime_dependency "rest-client"
+
+  s.add_dependency 'eventmachine'
+  s.add_dependency 'amqp'
+  s.add_dependency 'uuid'
+  s.add_dependency 'json'
+  s.add_dependency 'andand'
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: qswarm
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Mark Cheverton
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-29 00:00:00.000000000 +00:00
+default_executable: 
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &2153397540 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153397540
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: &2153397120 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153397120
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: &2153396700 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153396700
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &2153396280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153396280
+- !ruby/object:Gem::Dependency
+  name: andand
+  requirement: &2153395860 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *2153395860
+description: Framework for writing distributed agents hanging off an AMQP message
+  bus
+email:
+- mark.cheverton@ecafe.org
+executables:
+- qswarm
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/qswarm
+- lib/qswarm.rb
+- lib/qswarm/agent.rb
+- lib/qswarm/broker.rb
+- lib/qswarm/dsl.rb
+- lib/qswarm/listener.rb
+- lib/qswarm/loggable.rb
+- lib/qswarm/speaker.rb
+- lib/qswarm/swarm.rb
+- lib/qswarm/version.rb
+- qswarm.gemspec
+has_rdoc: true
+homepage: http://github.com/ennui2342/qswarm
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: qswarm
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: Distributed queue based agents in Ruby
+test_files: []