event-shipper 0.2.0 → 0.2.1

data/HISTORY

@@ -4,6 +4,13 @@
   + Proper framing of transmissions; transmissions are now versioned and 
     should work even in environments where server and client are not the same
     gem. This is the case for versions >= 0.2.
+  + Handles corrupt messages by dropping them. 
+  + Statistics output on SIGINFO; most OSs have that on Ctrl+T. 
+  + All events have the tag 'shipped'
+
+  . Infrastructure for annotating JSON hash with transport information. Not
+    so sure where to take this. 
+  . Infrastructure for sender-controlled queueing. Not sure if this is needed.
 
 = 0.1
 

data/lib/event_shipper/event.rb

@@ -13,7 +13,7 @@ module EventShipper
         :host => host, :path => path)
       @message = line.chomp.strip
       @fields = {}
-      @tags = []
+      @tags = %w(shipped)
     end
     
     def to_hash

data/lib/event_shipper/filter/encrypt.rb

@@ -80,7 +80,7 @@ module EventShipper::Filter
           h[user] = AES256.new(pwd); h }
     end
 
-    def de encrypted_string
+    def de encrypted_string, attributes={}
       message = parse_encrypted(encrypted_string)
 
       if algo = @user_db[message.user]
@@ -89,7 +89,9 @@ module EventShipper::Filter
           message.iv, 
           message.ciphertext)
 
-        return transmission
+        attributes[:encrypting_user] = message.user
+
+        return transmission, attributes
       else
         warn "No such user #{message.user} in database; cannot decrypt."
         return nil

data/lib/event_shipper/filter/transmission.rb

@@ -10,13 +10,10 @@ module EventShipper::Filter
   class Transmission
     include EventShipper::Protocol
 
-    def en hash
-      transmission(
-        event(
-          queue: 'queue', 
-          json: hash.to_json)).serialize_to_string
+    def en event
+      transmission(event).serialize_to_string
     end
-    def de string
+    def de string, attributes={}
       transmission = parse_transmission(string)
 
       if transmission.version != 1
@@ -24,7 +21,12 @@ module EventShipper::Filter
       end
 
       event = transmission.events.first
-      JSON.parse(event.json)
+      attributes[:queue] = event.queue
+
+      return JSON.parse(event.json), attributes
+
+    rescue ProtocolBuffers::DecodeError
+      return nil
     end
   end
 end

data/lib/event_shipper/protocol.rb

@@ -2,6 +2,8 @@
 require_relative 'protocol/v1.pb.rb'
 
 module EventShipper::Protocol
+
+module_function
   def transmission *events
     Transmission.new(
       version: 1, 

data/lib/event_shipper/proxy.rb

@@ -33,6 +33,8 @@ module EventShipper
         host: redis_config['host'], 
         port: redis_config['port'])
 
+      trap('SIGINFO') { puts transport.stats }
+
       transport.dispatch { |queue, message| 
         # puts message }
         redis.lpush queue, message }

data/lib/event_shipper/shipper.rb

@@ -31,6 +31,8 @@ module EventShipper
         tailer
       end
 
+      trap('SIGINFO') { puts transport.stats }
+
       # Block until everyone exits.
       tailers.each do |tailer|
         tailer.join

data/lib/event_shipper/stats.rb

@@ -0,0 +1,29 @@
+
+module EventShipper
+  class Stats
+    def initialize
+      reset
+    end
+
+    def count_message
+      @messages += 1
+    end
+    def count_failure
+      @failures += 1
+    end
+
+    def reset
+      @period_start = Time.now
+      @messages = @failures = 0
+    end
+
+    def to_s
+      seconds = Time.now - @period_start 
+
+      mps = @messages / Float(seconds)
+      fps = @failures / Float(seconds)
+
+      "#{mps} messages/s, #{fps} failures/s"
+    end
+  end
+end

data/lib/event_shipper/udp.rb

@@ -1,9 +1,12 @@
 require 'socket'
 
 require_relative 'filters'
+require_relative 'stats'
 
 module EventShipper
   class UDP
+    attr_reader :stats
+
     def initialize host, port
       @host, @port = host, port
       @socket = UDPSocket.new
@@ -13,18 +16,7 @@ module EventShipper
       @filters = []
       wrap Filter::Transmission.new
       
-      @messages_per_period = 0
-      @period_start = Time.now
-      Thread.start do
-        loop do
-          puts "total #{(@messages_per_period / (Time.now - @period_start)).round(3)} msgs/s"
-          
-          @messages_per_period = 0
-          @period_start = Time.now
-
-          sleep 10
-        end
-      end
+      @stats = Stats.new      
     end
 
     def wrap filter
@@ -38,24 +30,44 @@ module EventShipper
       @filters.reverse.inject(obj) { |o, f| f.de(o) }
     end
 
-    def send hash
-      @messages_per_period += 1
-      @socket.send encode(hash), 
+    # Sends messages via UDP to a central proxy. Using the queue argument, the
+    # caller can decide which redis queue events end up in. 
+    #
+    # @param hash [Hash] Attributes of the event to send. 
+    # @param queue [String] Queue to put events into on the other end
+    def send hash, queue='queue'
+      event = Protocol.event(
+        queue: queue, 
+        json: hash.to_json)
+
+      @socket.send encode(event), 
         0,    # flags...
         @host, @port
     end
 
+    # Handles a single message; decoding it and passing it to the caller. 
+    #
+    def handle_message string
+      msg, attributes = decode(string)
+
+      if msg
+        @stats.count_message
+
+        queue = attributes[:queue]
+        yield queue, msg
+      else
+        @stats.count_failure
+      end
+    end
+
     # Enters a loop, receiving messages, yielding them to the block. 
     #
     def dispatch
       @socket.bind @host, @port
       loop do
         datagram, source_info = @socket.recvfrom(10 * 1024)
-        @messages_per_period += 1
-
-        hash = decode(datagram)
 
-        yield nil, hash
+        handle_message(datagram)
       end
     end
   end

data/lib/stats.rb

@@ -0,0 +1,29 @@
+
+module EventShipper
+  class Stats
+    def initialize
+      reset
+    end
+
+    def count_message
+      @messages += 1
+    end
+    def count_failure
+      @failures += 1
+    end
+
+    def reset
+      @period_start = Time.now
+      @messages = @failures = 0
+    end
+
+    def to_s
+      seconds = Time.now - @period_start 
+
+      mps = @messages / Float(seconds)
+      fps = @failures / Float(seconds)
+
+      "#{mps} messages/s, #{fps} failures/s"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: event-shipper
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
   prerelease: 
 platform: ruby
 authors:
@@ -131,8 +131,10 @@ files:
 - lib/event_shipper/protocol.rb
 - lib/event_shipper/proxy.rb
 - lib/event_shipper/shipper.rb
+- lib/event_shipper/stats.rb
 - lib/event_shipper/udp.rb
 - lib/event_shipper.rb
+- lib/stats.rb
 - bin/bench
 - bin/esproxy
 - bin/esshipper