dripdrop 0.0.2 → 0.0.3

data/README.md

@@ -3,9 +3,25 @@
 0MQ Based App Event Monitoring / processing.
 A work in progress.
 
+# Why use dripdrop?
+
+You want to asynchronously process / monitor arbitrary messages from your app.
+dripdrop does this well for a few reasons.
+
+* It's fast. dripdrop doesn't slow down your app. 0MQ + Bert are fast. Sending a message never blocks.
+* It's flexible. By leveraging 0MQ pub/sub sockets you can have many different processors (collectors in dripdrop) that don't impact or even care about each other
+* It's easy. Check out the agent and collector examples below. You can be processing stuff in no time.
+
 ## An example with a WebSocket UI:
 
-To run a simple example, feeding data to a websockets UI
+### You'll need to have the zmq dev libs on your machine. On OSX this means
+
+1. Download and build zeromq from [zeromq.org](http://www.zeromq.org/area:download)
+1. The agent just uses the plain zmq gem, which runs fine on ruby 1.8.7+, *EDIT!* If you use my fork of ffi-rzmq everything should work with 1.8.7! This isn't fully tested yet though *END EDIT*  this is so you can use it in say your rails app. Everything else needs ruby 1.9.2 or jruby and uses Chuck Remes [ffi-rzmq](http://github.com/chuckremes/ffi-rzmq), and [zmqmachine](http://github.com/chuckremes/zmqmachine) gems which you must build yourself. I recommend using rvm to enable the use of multiple rubies on one machine.
+1. zmq_forwarder comes with zmq, use this to aggregate agent messages using the example config shown below
+
+### To run a simple example, feeding data to a websockets UI
+
 #### Aggregate agents with zmq_forwarder (comes with zmq)
     $ zmq_forwarder examples/forwarder.cfg
 
@@ -21,6 +37,8 @@ To run a simple example, feeding data to a websockets UI
 
 ## Example Topology
 
+You can add as many listeners as you want, or reconfigure things any way you want. Heres how I plan on using it.
+
 ![topology](http://github.com/andrewvc/dripdrop/raw/master/doc_img/topology.png "Topology")
 
 ## Sending Messages

data/Rakefile

@@ -18,26 +18,6 @@ rescue LoadError
   puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
 end
 
-require 'rake/testtask'
-Rake::TestTask.new(:test) do |test|
-  test.libs << 'lib' << 'test'
-  test.pattern = 'test/**/test_*.rb'
-  test.verbose = true
-end
-
-begin
-  require 'rcov/rcovtask'
-  Rcov::RcovTask.new do |test|
-    test.libs << 'test'
-    test.pattern = 'test/**/test_*.rb'
-    test.verbose = true
-  end
-rescue LoadError
-  task :rcov do
-    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
-  end
-end
-
 task :test => :check_dependencies
 
 task :default => :test

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/dripdrop.gemspec

@@ -5,14 +5,14 @@
 
 Gem::Specification.new do |s|
   s.name = %q{dripdrop}
-  s.version = "0.0.2"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Cholakian"]
   s.date = %q{2010-08-14}
   s.description = %q{0MQ App stats}
   s.email = %q{andrew@andrewvc.com}
-  s.executables = ["drip-collector", "drip-mlogger", "drip-publisher"]
+  s.executables = ["drip-mlogger", "drip-publisher"]
   s.extra_rdoc_files = [
     "LICENSE",
      "README.md"
@@ -24,7 +24,6 @@ Gem::Specification.new do |s|
      "README.md",
      "Rakefile",
      "VERSION",
-     "bin/drip-collector",
      "bin/drip-mlogger",
      "bin/drip-publisher",
      "doc_img/topology.png",
@@ -41,8 +40,7 @@ Gem::Specification.new do |s|
      "lib/dripdrop/collector.rb",
      "lib/dripdrop/message.rb",
      "lib/dripdrop/mlogger.rb",
-     "lib/dripdrop/publisher.rb",
-     "lib/dripdrop/webserver.rb"
+     "lib/dripdrop/publisher.rb"
   ]
   s.homepage = %q{http://github.com/andrewvc/dripdrop}
   s.rdoc_options = ["--charset=UTF-8"]

data/lib/dripdrop/agent.rb

@@ -3,8 +3,11 @@ require 'zmq'
 require 'bert'
 
 class DripDrop
+  #The Agent class is a simple ZMQ Pub client. It uses DripDrop::Message messages
   class Agent
     attr_reader :address, :context, :socket
+    
+    #address should be a string like tcp://127.0.0.1
     def initialize(address)
       @address = address
       @context = ZMQ::Context.new(1)
@@ -12,8 +15,9 @@ class DripDrop
       @socket.connect(@address)
     end
 
-    def send_message(name,content)
-      puts @socket.send(Message.new(name,content).encoded, 0)
+    #Sends a DripDrop::Message to the socket
+    def send_message(name,body,head={})
+      puts @socket.send(DripDrop::Message.new(name,:body => body, :head => head).encoded, 0)
     end
   end
 end

data/lib/dripdrop/collector.rb

@@ -4,9 +4,12 @@ require 'uri'
 require 'dripdrop/message'
 
 class DripDrop
+  #Publishes the ZMQ messages. This is not evented as zmqmachine seems
+  #to max out the CPU on ZMQ::PUB sockets
   class CollectorPub
     attr_reader :context, :socket, :address
     
+    #Takes either as string or URI as an address
     def initialize(address)
       @address = address
       @context = ZMQ::Context.new(1)
@@ -14,14 +17,17 @@ class DripDrop
       @socket.bind(@address.to_s)
     end
     
+    #Sends an already encoded DripDrop::Message
     def send_message(message)
       @socket.send(message)
     end
   end
    
+  #Listens on a zmqmachine sub_socket.
   class CollectorSub
     attr_reader :context, :socket, :address, :collector
     
+    #Takes a zmqmachine reactor in sub mode's context, self, address, and a CollectorPublisher
     def initialize(context, collector, address, publisher=nil)
       @context   = context
       @collector = collector
@@ -40,9 +46,13 @@ class DripDrop
       end
     end
   end
-  
+
+  #Collector is meant to be subclassed. It's used to provide basic pub/sub functionality.
+  #Subclasses should provide +on_recv+, which gets called on receipt of a message.
+  #If pub_addr is specified +publish+ can be called, which sends a message to the pub socket.
   class Collector
     attr_reader :sub_reactor, :sub_addr, :pub_addr
+    
     def initialize(sub_addr='tcp://127.0.0.1:2900',pub_addr=nil)
       sub_addr_uri = URI.parse(sub_addr)
       host, port   = sub_addr_uri.host, sub_addr_uri.port.to_i
@@ -56,6 +66,7 @@ class DripDrop
       end
     end
 
+    #Run the collector. Returns the reactor, so if this is the only thing in your script, be sure to call +join+ on the reactor.
     def run
       puts "Run"
       @sub_reactor.run do |context|
@@ -64,10 +75,13 @@ class DripDrop
       @sub_reactor
     end
 
+    #If pub_addr was specified when the collector was initialized, messages can be broadcast using this.
     def publish(message)
       @publisher.send_string(message.encoded)
     end
      
+    #Intended to be overriden by a subclass.
+    #Receives an encoded DripDrop::Message
     def on_recv(message); end
   end
 end

data/lib/dripdrop/message.rb

@@ -2,9 +2,20 @@ require 'rubygems'
 require 'bert'
 
 class DripDrop
+  #DripDrop::Message messages are exchanged between all tiers in the architecture
+  #A Message is composed of a name, head, and body. The name exists primarily for the
+  #purpose of native ZMQ filtering, since ZMQ can filter based on a message prefix.
+  #
+  #The name is any string consisting of non-null chars.
+  #The rest of the payload is a BERT encoded head and body, both of which are hashes.
+  #The head and body don't have rigid definitions yet, use as you please.
   class Message
     attr_accessor :name, :head, :body
     
+    #Create a new message.
+    #example:
+    #  Message.new('mymessage', :head => {:timestamp => Time.now}, 
+    #    :body => {:mykey => :myval,  :other_key => ['complex']})
     def initialize(name,extra={})
       raise "No null chars allowed in message names!" if name.include?("\0")
        
@@ -15,14 +26,18 @@ class DripDrop
       @body = extra[:body]
     end
     
+    #The encoded message, ready to be sent across the wire via ZMQ
     def encoded
       "#{@name}\0#{BERT.encode({:head => @head, :body => @body})}"
     end
     
+    #Convert the Message to a hash like:
+    #{:name => @name, :head => @head, :body => @body}
     def to_hash
       {:name => @name, :head => @head, :body => @body}
     end
 
+    #Parses an encoded message
     def self.parse(msg)
       name, encoded_body = msg.split("\0",2)
       decoded = BERT.decode(encoded_body)

data/lib/dripdrop/mlogger.rb

@@ -13,6 +13,7 @@ class DripDrop
     end
   end
 
+  #MongoDB Logger for DripDrop::Message messages
   class MLogger
     attr_reader :sub_address, :sub_reactor, :mongo_host, :mongo_port, :mongo_db,
                 :mongo_connection, :mongo_collection

data/lib/dripdrop/publisher.rb

@@ -24,8 +24,11 @@ class DripDrop
     end
   end
 
+
+  #WebSocket server that rebroadcasts all DripDrop::Messages it subscribes to as JSON
   class Publisher
     attr_reader :sub_address, :sub_collector, :ws_address
+     
     def initialize(sub_address='tcp://127.0.0.1:2901',ws_address='ws://127.0.0.1:2902')
       @sub_address   = URI.parse(sub_address)
       @ws_address    = URI.parse(ws_address)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 3
+  version: 0.0.3
 platform: ruby
 authors: 
 - Andrew Cholakian
@@ -46,7 +46,6 @@ dependencies:
 description: 0MQ App stats
 email: andrew@andrewvc.com
 executables: 
-- drip-collector
 - drip-mlogger
 - drip-publisher
 extensions: []
@@ -61,7 +60,6 @@ files:
 - README.md
 - Rakefile
 - VERSION
-- bin/drip-collector
 - bin/drip-mlogger
 - bin/drip-publisher
 - doc_img/topology.png
@@ -79,7 +77,6 @@ files:
 - lib/dripdrop/message.rb
 - lib/dripdrop/mlogger.rb
 - lib/dripdrop/publisher.rb
-- lib/dripdrop/webserver.rb
 has_rdoc: true
 homepage: http://github.com/andrewvc/dripdrop
 licenses: []

data/bin/drip-collector

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'dripdrop/collector'
-
-puts "Starting Collector"
-DripDrop::Collector.new.run
-puts "Ended"

data/lib/dripdrop/webserver.rb

@@ -1,10 +0,0 @@
-require 'rubygems'
-require 'sinatra'
-
-set :static, true
-set :logging, false
-
-get '/' do
-  puts "Started Sinatra"
-  File.open('public/view.html')
-end