dripdrop 0.3.1 → 0.4.0

data/README.md

@@ -4,48 +4,43 @@
 
 DripDrop is ZeroMQ(using zmqmachine) + Event Machine simplified for the general use case + serialization helpers.
 
-Here's an example of the kind of thing DripDrop makes easy, from [examples/pubsub.rb](http://github.com/andrewvc/dripdrop/blob/master/example/pubsub.rb)
- 
-    require 'dripdrop/node'
-    Thread.abort_on_exception = true
-
-    #Define our handlers
+Here's an example of the kind of thing DripDrop makes easy, from [example/combined.rb](http://github.com/andrewvc/dripdrop/blob/master/example/combined.rb)
+  
+    require 'dripdrop'
+    Thread.abort_on_exception = true #Always a good idea in multithreaded apps.
+  
+    # Encapsulates our EM and ZMQ reactors
     DripDrop::Node.new do
-      z_addr = 'tcp://127.0.0.1:2200'
+      # Define all our sockets
+      route :stats_pub,      :zmq_publish,   'tcp://127.0.0.1:2200', :bind
+      route :stats_sub1,     :zmq_subscribe, stats_pub.address, :connect
+      route :stats_sub2,     :zmq_subscribe, stats_pub.address, :connect
+      route :http_collector, :http_server,   'http://127.0.0.1:8080'
+      route :http_agent,     :http_client,   http_collector.address
         
-      #Create a publisher
-      pub = zmq_publish(z_addr,:bind)
-
-      #Create two subscribers
-      zmq_subscribe(z_addr,:connect).on_recv do |message|
-        puts "Receiver 1 #{message.inspect}"
+      stats_sub1.on_recv do |message|
+        puts "Receiver 1: #{message.body}"
       end
-      zmq_subscribe(z_addr, :connect).on_recv do |message|
-        puts "Receiver 2 #{message.inspect}"
+      stats_sub2.on_recv do |message|
+        puts "Receiver 2: #{message.body}"
       end
       
-      zm_reactor.periodical_timer(5) do
-        #Sending a hash as a message implicitly transforms it into a DripDrop::Message
-        pub.send_message(:name => 'test', :body => 'Test Payload')
-      end
-      
-      http_server(addr).on_recv do |msg,response|
+      i = 0
+      http_collector.on_recv do |message,response|
         i += 1
-        response.send_message(msg)
+        stats_pub.send_message(message)
+        response.send_message(:name => 'ack', :body => {:seq => i})
       end
 
       EM::PeriodicTimer.new(1) do
-        client = http_client(addr)
         msg = DripDrop::Message.new('http/status', :body => "Success #{i}")
-        client.send_message(msg) do |resp_msg|
-          puts resp_msg.inspect
+        http_agent.send_message(msg) do |resp_msg|
+          puts "RESP: #{resp_msg.body['seq']}"
         end
       end
     end.start! #Start the reactor and block until complete
 
-Note that these aren't regular ZMQ sockets, and that the HTTP server isn't a regular server. They only speak and respond using DripDrop::Message formatted messages. For HTTP/WebSockets it's JSON that looks like {name: 'name', head: {}, body: anything}, for ZeroMQ it means BERT. There is a raw made that you can use for other message formats, but using DripDrop::Messages makes things easier, and for some socket types (like XREQ/XREP) the predefined format is very useful in matching requests to replies.
-
-Want to see a longer example encapsulating both zmqmachine and eventmachine functionality? Check out [this file](http://github.com/andrewvc/dripdrop-webstats/blob/master/lib/dripdrop-webstats.rb).
+Note that these aren't regular ZMQ sockets, and that the HTTP server isn't a regular server. They only speak and respond using DripDrop::Message formatted messages. For HTTP/WebSockets it's JSON that looks like {name: 'name', head: {}, body: anything}, for ZeroMQ it means BERT. There is a raw mode that you can use for other message formats, but using DripDrop::Messages makes things easier, and for some socket types (like XREQ/XREP) the predefined format is very useful in matching requests to replies.
 
 #RDoc
 

data/VERSION

@@ -1 +1 @@
-0.3.1
+0.4.0

data/dripdrop.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{dripdrop}
-  s.version = "0.3.1"
+  s.version = "0.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrew Cholakian"]
-  s.date = %q{2010-10-21}
+  s.date = %q{2010-11-15}
   s.description = %q{Evented framework for ZeroMQ and EventMachine Apps. }
   s.email = %q{andrew@andrewvc.com}
   s.extra_rdoc_files = [
@@ -26,25 +26,43 @@ Gem::Specification.new do |s|
      "doc_img/topology.png",
      "dripdrop.gemspec",
      "example/agent_test.rb",
+     "example/combined.rb",
      "example/http.rb",
      "example/pubsub.rb",
      "example/pushpull.rb",
+     "example/stats_app/core.rb",
+     "example/stats_app/public/.sass-cache/b48b4299d80c05f528daf63fe51d85e5e3c10d98/stats.scssc",
+     "example/stats_app/public/backbone.js",
+     "example/stats_app/public/build_templates.rb",
+     "example/stats_app/public/json2.js",
+     "example/stats_app/public/protovis-r3.2.js",
+     "example/stats_app/public/stats.css",
+     "example/stats_app/public/stats.haml",
+     "example/stats_app/public/stats.html",
+     "example/stats_app/public/stats.js",
+     "example/stats_app/public/stats.scss",
+     "example/stats_app/public/underscore.js",
      "example/subclass.rb",
      "example/xreq_xrep.rb",
      "js/dripdrop.html",
      "js/dripdrop.js",
-     "js/jack.js",
      "js/qunit.css",
      "js/qunit.js",
      "lib/dripdrop.rb",
      "lib/dripdrop/agent.rb",
+     "lib/dripdrop/handlers/base.rb",
      "lib/dripdrop/handlers/http.rb",
      "lib/dripdrop/handlers/websockets.rb",
      "lib/dripdrop/handlers/zeromq.rb",
      "lib/dripdrop/message.rb",
      "lib/dripdrop/node.rb",
+     "lib/dripdrop/node/nodelet.rb",
+     "spec/gimite-websocket.rb",
      "spec/message_spec.rb",
      "spec/node/http_spec.rb",
+     "spec/node/nodelet_spec.rb",
+     "spec/node/routing_spec.rb",
+     "spec/node/websocket_spec.rb",
      "spec/node/zmq_pushpull_spec.rb",
      "spec/node/zmq_xrepxreq_spec.rb",
      "spec/node_spec.rb",
@@ -57,9 +75,13 @@ Gem::Specification.new do |s|
   s.summary = %q{Evented framework for ZeroMQ and EventMachine Apps.}
   s.test_files = [
     "spec/spec_helper.rb",
+     "spec/gimite-websocket.rb",
      "spec/node/http_spec.rb",
+     "spec/node/routing_spec.rb",
      "spec/node/zmq_xrepxreq_spec.rb",
      "spec/node/zmq_pushpull_spec.rb",
+     "spec/node/nodelet_spec.rb",
+     "spec/node/websocket_spec.rb",
      "spec/message_spec.rb",
      "spec/node_spec.rb"
   ]

data/example/combined.rb

@@ -0,0 +1,33 @@
+require 'dripdrop'
+Thread.abort_on_exception = true #Always a good idea in multithreaded apps.
+
+# Encapsulates our EM and ZMQ reactors
+DripDrop::Node.new do
+  # Define all our sockets
+  route :stats_pub,      :zmq_publish,   'tcp://127.0.0.1:2200', :bind
+  route :stats_sub1,     :zmq_subscribe, stats_pub.address, :connect
+  route :stats_sub2,     :zmq_subscribe, stats_pub.address, :connect
+  route :http_collector, :http_server,   'http://127.0.0.1:8080'
+  route :http_agent,     :http_client,   http_collector.address
+    
+  stats_sub1.on_recv do |message|
+    puts "Receiver 1: #{message.body}"
+  end
+  stats_sub2.on_recv do |message|
+    puts "Receiver 2: #{message.body}"
+  end
+  
+  i = 0
+  http_collector.on_recv do |message,response|
+    i += 1
+    stats_pub.send_message(message)
+    response.send_message(:name => 'ack', :body => {:seq => i})
+  end
+
+  EM::PeriodicTimer.new(1) do
+    msg = DripDrop::Message.new('http/status', :body => "Success #{i}")
+    http_agent.send_message(msg) do |resp_msg|
+      puts "RESP: #{resp_msg.body['seq']}"
+    end
+  end
+end.start! #Start the reactor and block until complete

data/example/pubsub.rb

@@ -3,33 +3,25 @@ Thread.abort_on_exception = true
 
 #Define our handlers
 DripDrop::Node.new do
-  z_addr = 'tcp://127.0.0.1:2200'
+  route :pub,  :zmq_publish,   'tcp://127.0.0.1:2200', :bind
+  route :sub1, :zmq_subscribe, pub.address, :connect, :topic_filter => /[13579]$/
+  route :sub2, :zmq_subscribe, pub.address, :connect, :topic_filter => /[02468]$/
+  route :sub3, :zmq_subscribe, pub.address, :connect
     
-  #Create a publisher
-  pub = zmq_publish(z_addr,:bind)
-
-  #Create three subscribers
-  sub1 = zmq_subscribe(z_addr,:connect)
-
   sub1.on_recv do |message|
     puts "Receiver 1 #{message.inspect}"
   end
 
-  sub1.topic_filter = /[13579]$/
-
-  sub2 = zmq_subscribe(z_addr,:connect)
-
   sub2.on_recv do |message|
     puts "Receiver 2 #{message.inspect}"
   end
 
-  sub2.topic_filter = /[02468]$/
-
-  zmq_subscribe(z_addr, :connect).on_recv do |message|
+  sub3.on_recv do |message|
     puts "Receiver 3 #{message.inspect}"
   end
   
-  zm_reactor.periodical_timer(5) do
+  zm_reactor.periodical_timer(500) do
+    puts "Sending!"
     #Sending a hash as a message implicitly transforms it into a DripDrop::Message
     pub.send_message(:name => Time.now.to_i.to_s, :body => 'Test Payload')
   end

data/example/stats_app/core.rb

@@ -0,0 +1,113 @@
+require 'dripdrop'
+Thread.abort_on_exception = true #Always a good idea in multithreaded apps.
+
+# This demo app is an message stats application
+# It receives stats data via either HTTP or ZMQ directly, aggregates,
+# and keeps track of data.
+DripDrop::Node.new do
+  routes_for :agg do
+    route :input,  :zmq_subscribe, 'tcp://127.0.0.1:2200', :bind
+    route :output, :zmq_publish,   'tcp://127.0.0.1:2201', :bind
+    route :input_http, :http_server, 'http://127.0.0.1:8082'
+  end
+ 
+  routes_for :counter do
+    route :input,      :zmq_subscribe, agg_output.address, :connect
+    route :query,      :zmq_xrep,      'tcp://127.0.0.1:2203', :bind
+    route :query_http, :http_server,   'tcp://0.0.0.0:8081'
+  end
+
+  routes_for :tracer do
+    route :input,  :zmq_subscribe, agg_output.address, :connect, :topic_filter => /^ip_trace_req$/
+    route :output, :zmq_publish, 'tcp://127.0.0.1:2204', :bind
+  end
+  
+  routes_for :ws_stream do
+    route :tracer_input, :zmq_subscribe, agg_output.address,    :connect
+    route :agg_input,    :zmq_subscribe, tracer_output.address, :connect
+    route :client,       :websocket,     'ws://127.0.0.1:2202'
+  end
+  
+  routes_for :heartbeat do
+    route :output, :zmq_publish, agg_input.address, :connect
+  end
+
+  nodelet :agg do |agg|
+    agg.input.on_recv do |message|
+      agg.output.send_message(message)
+    end
+    
+    agg.input.on_recv do |message|
+      agg.output.send_message(message)
+    end
+    
+    agg.input_http.on_recv do |message,response,env|
+      response.send_message(:name => 'ack')
+      agg.output.send_message(message)
+    end
+  end 
+
+  nodelet :counter do |cntr|
+    stats = {:total => 0, :name_counts => Hash.new(0) }
+    
+    cntr.input.on_recv do |message|
+      stats[:total] += 1
+      stats[:name_counts][message.name] += 1
+    end
+    
+    cntr.query.on_recv do |message,ids,seq|
+      cntr.query.send_message({:name => 'stats', :body => @stats}, ids, seq)
+    end
+
+    cntr.query_http.on_recv do |message,response|
+      response.send_message(:name => 'stats', :body => @stats)
+    end
+  end
+
+  nodelet :tracer do |tracer|
+    tracer_memo = {}
+    
+    ip_regexp = /\A(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\Z/
+    tracer.input.on_recv do |message|
+      puts "TRACE #{message.body.inspect}"
+       
+      ip = message.body['ip']
+      puts "IP #{message.inspect}"
+      if ip =~ ip_regexp
+        memoized_res = tracer_memo[ip]
+        if memoized_res
+          tracer.output.send_message(:name => 'ip_route', :body => {:ip => ip, :route => memoized_res})
+        else
+          EM.system("/usr/sbin/traceroute -w 4 #{ip}") do |output,status|
+            route = output.split("\n")[1..-1].map {|l| l.split(/ /)[3] }.select {|a| a != '*'}
+            tracer_memo[ip] = route
+            tracer.output.send_message(:name => 'ip_route', :body => {:ip => ip, :route => route})
+          end
+        end
+      end
+    end
+  end
+
+  nodelet :ws_stream do |wss|
+    [wss.tracer_input, wss.agg_input].each do |input|
+      input.on_recv do |message|
+        send_internal(:wss, message)
+      end
+    end
+    
+    wss.client.on_open do |ws|
+      recv_internal(:wss, ws.signature) do |message|
+        ws.send_message(message)
+      end
+    end.on_recv do |message,ws|
+    end.on_close do |ws|
+    end.on_error do |ws|
+    end
+  end
+  
+  nodelet :heartbeat do |hbeat|
+    zm_reactor.periodical_timer(1000) do
+      hbeat.output.send_message(:name => 'heartbeat/tick', :body => Time.now.to_i)
+    end
+  end
+end.start! #Start the reactor and block until complete

data/example/stats_app/public/backbone.js

@@ -0,0 +1,16 @@
+(function(){var f;f=typeof exports!=="undefined"?exports:this.Backbone={};f.VERSION="0.2.0";var e=this._;if(!e&&typeof require!=="undefined")e=require("underscore")._;var h=this.jQuery;f.emulateHttp=false;f.Events={bind:function(a,b){this._callbacks||(this._callbacks={});(this._callbacks[a]||(this._callbacks[a]=[])).push(b);return this},unbind:function(a,b){var c;if(a){if(c=this._callbacks)if(b){c=c[a];if(!c)return this;for(var d=0,g=c.length;d<g;d++)if(b===c[d]){c.splice(d,1);break}}else c[a]=[]}else this._callbacks=
+{};return this},trigger:function(a){var b,c,d,g;if(!(c=this._callbacks))return this;if(b=c[a]){d=0;for(g=b.length;d<g;d++)b[d].apply(this,Array.prototype.slice.call(arguments,1))}if(b=c.all){d=0;for(g=b.length;d<g;d++)b[d].apply(this,arguments)}return this}};f.Model=function(a){this.attributes={};this.cid=e.uniqueId("c");this.set(a||{},{silent:true});this._previousAttributes=e.clone(this.attributes);this.initialize&&this.initialize(a)};e.extend(f.Model.prototype,f.Events,{_previousAttributes:null,
+_changed:false,toJSON:function(){return e.clone(this.attributes)},get:function(a){return this.attributes[a]},set:function(a,b){b||(b={});if(!a)return this;if(a.attributes)a=a.attributes;var c=this.attributes;if(this.validate){var d=this.validate(a);if(d){b.error?b.error(this,d):this.trigger("error",this,d);return false}}if("id"in a)this.id=a.id;for(var g in a){d=a[g];if(d==="")d=null;if(!e.isEqual(c[g],d)){c[g]=d;if(!b.silent){this._changed=true;this.trigger("change:"+g,this,d)}}}!b.silent&&this._changed&&
+this.change();return this},unset:function(a,b){b||(b={});var c=this.attributes[a];delete this.attributes[a];if(!b.silent){this._changed=true;this.trigger("change:"+a,this);this.change()}return c},fetch:function(a){a||(a={});var b=this,c=a.error&&e.bind(a.error,null,b);f.sync("read",this,function(d){if(!b.set(b.parse(d),a))return false;a.success&&a.success(b,d)},c);return this},save:function(a,b){a||(a={});b||(b={});if(!this.set(a,b))return false;var c=this,d=b.error&&e.bind(b.error,null,c),g=this.isNew()?
+"create":"update";f.sync(g,this,function(i){if(!c.set(c.parse(i),b))return false;b.success&&b.success(c,i)},d);return this},destroy:function(a){a||(a={});var b=this,c=a.error&&e.bind(a.error,null,b);f.sync("delete",this,function(d){b.collection&&b.collection.remove(b);a.success&&a.success(b,d)},c);return this},url:function(){var a=j(this.collection);if(this.isNew())return a;return a+"/"+this.id},parse:function(a){return a},clone:function(){return new this.constructor(this)},isNew:function(){return!this.id},
+change:function(){this.trigger("change",this);this._previousAttributes=e.clone(this.attributes);this._changed=false},hasChanged:function(a){if(a)return this._previousAttributes[a]!=this.attributes[a];return this._changed},changedAttributes:function(a){a||(a=this.attributes);var b=this._previousAttributes,c=false,d;for(d in a)if(!e.isEqual(b[d],a[d])){c=c||{};c[d]=a[d]}return c},previous:function(a){if(!a||!this._previousAttributes)return null;return this._previousAttributes[a]},previousAttributes:function(){return e.clone(this._previousAttributes)}});
+f.Collection=function(a,b){b||(b={});if(b.comparator){this.comparator=b.comparator;delete b.comparator}this._boundOnModelEvent=e.bind(this._onModelEvent,this);this._reset();a&&this.refresh(a,{silent:true});this.initialize&&this.initialize(a,b)};e.extend(f.Collection.prototype,f.Events,{model:f.Model,toJSON:function(){return this.map(function(a){return a.toJSON()})},add:function(a,b){if(e.isArray(a))for(var c=0,d=a.length;c<d;c++)this._add(a[c],b);else this._add(a,b);return this},remove:function(a,
+b){if(e.isArray(a))for(var c=0,d=a.length;c<d;c++)this._remove(a[c],b);else this._remove(a,b);return this},get:function(a){return a&&this._byId[a.id!=null?a.id:a]},getByCid:function(a){return a&&this._byCid[a.cid||a]},at:function(a){return this.models[a]},sort:function(a){a||(a={});if(!this.comparator)throw Error("Cannot sort a set without a comparator");this.models=this.sortBy(this.comparator);a.silent||this.trigger("refresh",this);return this},pluck:function(a){return e.map(this.models,function(b){return b.get(a)})},
+refresh:function(a,b){a||(a=[]);b||(b={});this._reset();this.add(a,{silent:true});b.silent||this.trigger("refresh",this);return this},fetch:function(a){a||(a={});var b=this,c=a.error&&e.bind(a.error,null,b);f.sync("read",this,function(d){b.refresh(b.parse(d));a.success&&a.success(b,d)},c);return this},create:function(a,b){b||(b={});a instanceof f.Model||(a=new this.model(a));var c=a.collection=this;return a.save(null,{success:function(d,g){c.add(d);b.success&&b.success(d,g)},error:b.error})},parse:function(a){return a},
+chain:function(){return e(this.models).chain()},_reset:function(){this.length=0;this.models=[];this._byId={};this._byCid={}},_add:function(a,b){b||(b={});a instanceof f.Model||(a=new this.model(a));var c=this.getByCid(a);if(c)throw Error(["Can't add the same model to a set twice",c.id]);this._byId[a.id]=a;this._byCid[a.cid]=a;a.collection=this;this.models.splice(this.comparator?this.sortedIndex(a,this.comparator):this.length,0,a);a.bind("all",this._boundOnModelEvent);this.length++;b.silent||this.trigger("add",
+a);return a},_remove:function(a,b){b||(b={});a=this.getByCid(a);if(!a)return null;delete this._byId[a.id];delete this._byCid[a.cid];delete a.collection;this.models.splice(this.indexOf(a),1);a.unbind("all",this._boundOnModelEvent);this.length--;b.silent||this.trigger("remove",a);return a},_onModelEvent:function(a,b){if(a==="change:id"){delete this._byId[b.previous("id")];this._byId[b.id]=b}this.trigger.apply(this,arguments)}});e.each(["forEach","each","map","reduce","reduceRight","find","detect","filter",
+"select","reject","every","all","some","any","include","invoke","max","min","sortBy","sortedIndex","toArray","size","first","rest","last","without","indexOf","lastIndexOf","isEmpty"],function(a){f.Collection.prototype[a]=function(){return e[a].apply(e,[this.models].concat(e.toArray(arguments)))}});f.View=function(a){this._configure(a||{});this._ensureElement();this.delegateEvents();this.initialize&&this.initialize(a)};var k=function(a){return h(a,this.el)},l=/^(\w+)\s*(.*)$/;e.extend(f.View.prototype,
+{tagName:"div",$:k,jQuery:k,render:function(){return this},make:function(a,b,c){a=document.createElement(a);b&&h(a).attr(b);c&&h(a).html(c);return a},delegateEvents:function(a){if(!(a||(a=this.events)))return this;h(this.el).unbind();for(var b in a){var c=a[b],d=b.match(l),g=d[1];d=d[2];c=e.bind(this[c],this);d===""?h(this.el).bind(g,c):h(this.el).delegate(d,g,c)}return this},_configure:function(a){if(this.options)a=e.extend({},this.options,a);if(a.model)this.model=a.model;if(a.collection)this.collection=
+a.collection;if(a.el)this.el=a.el;if(a.id)this.id=a.id;if(a.className)this.className=a.className;if(a.tagName)this.tagName=a.tagName;this.options=a},_ensureElement:function(){if(!this.el){var a={};if(this.id)a.id=this.id;if(this.className)a.className=this.className;this.el=this.make(this.tagName,a)}}});var n=f.Model.extend=f.Collection.extend=f.View.extend=function(a,b){var c=m(this,a,b);c.extend=n;return c},o={create:"POST",update:"PUT","delete":"DELETE",read:"GET"};f.sync=function(a,b,c,d){var g=
+a==="create"||a==="update"?{model:JSON.stringify(b)}:{};a=o[a];if(f.emulateHttp&&(a==="PUT"||a==="DELETE")){g._method=a;a="POST"}h.ajax({url:j(b),type:a,data:g,dataType:"json",success:c,error:d})};var m=function(a,b,c){var d;d=b.hasOwnProperty("constructor")?b.constructor:function(){return a.apply(this,arguments)};var g=function(){};g.prototype=a.prototype;d.prototype=new g;e.extend(d.prototype,b);c&&e.extend(d,c);return d.prototype.constructor=d},j=function(a){if(!(a&&a.url))throw Error("A 'url' property or function must be specified");
+return e.isFunction(a.url)?a.url():a.url}})();

data/example/stats_app/public/build_templates.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+#Compile the HAML and SASS templates
+
+`haml stats.haml > stats.html`
+`sass stats.scss > stats.css`