event-shipper 0.1.1 → 0.2.0

data/HISTORY

@@ -1,4 +1,10 @@
 
+= 0.2
+  
+  + Proper framing of transmissions; transmissions are now versioned and 
+    should work even in environments where server and client are not the same
+    gem. This is the case for versions >= 0.2.
+
 = 0.1
 
   * First version.

data/bin/bench

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# Benchmarks message code. 
+
+require 'bundler/setup'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'event_shipper'
+require 'event_shipper/benchmark'
+
+EventShipper::Benchmark.run

data/bin/esproxy

@@ -6,6 +6,6 @@
 require 'bundler/setup'
 
 $:.unshift File.dirname(__FILE__) + "/../lib"
-require 'event_shipper/proxy'
+require 'event_shipper'
 
 EventShipper::Proxy.run

data/bin/esshipper

@@ -6,6 +6,6 @@
 require 'bundler/setup'
 
 $:.unshift File.dirname(__FILE__) + "/../lib"
-require 'event_shipper/shipper'
+require 'event_shipper'
 
 EventShipper::Shipper.run

data/lib/event_shipper.rb

@@ -0,0 +1,6 @@
+
+
+module EventShipper; end
+
+require 'event_shipper/proxy'
+require 'event_shipper/shipper'

data/lib/event_shipper/benchmark.rb

@@ -0,0 +1,46 @@
+
+require 'benchmark'
+require 'clamp'
+
+module EventShipper
+  class Benchmark < Clamp::Command
+    def initialize *args
+      super
+
+      @s = UDP.new('localhost', 5050)
+      @s.wrap Filter::Encrypt.new('user', 'password')
+
+      @r = UDP.new('localhost', 5050)
+      @r.wrap Filter::Decrypt.new('user' => 'password')
+    end
+
+    def execute
+      puts "Simulates 1000 times the sending of a simple message: "
+      measure do
+        1000.times do _send_message end
+      end
+
+      puts "Simulates 1000 times the receiving of a message: "
+      msg = _send_message
+      measure do
+        1000.times do _receive_message(msg) end
+      end
+    end
+
+    def measure
+      puts ::Benchmark.measure {
+        yield
+      }
+    end
+
+    def _send_message
+      @s.encode(
+        source: 'benchmark', 
+        event: 'this is the event')
+    end
+
+    def _receive_message msg
+      @r.decode(msg)
+    end
+  end
+end

data/lib/event_shipper/event.rb

@@ -1,5 +1,4 @@
 
-require 'yajl'
 require 'addressable/uri'
 
 module EventShipper
@@ -29,13 +28,5 @@ module EventShipper
         '@message' => @message
       }
     end
-    
-    def to_json
-      Yajl::Encoder.encode(to_hash)
-    end
-    
-    def to_s
-      to_json
-    end
   end
 end

data/lib/event_shipper/{transport → filter}/encrypt.rb

@@ -1,9 +1,10 @@
 
 require 'openssl'
 
-module EventShipper; end
-module EventShipper::Transport
-  class Encryption
+require 'event_shipper/protocol'
+
+module EventShipper::Filter
+  class AES256
     def initialize password
       @password = password
       @salt = generate_salt
@@ -36,13 +37,9 @@ module EventShipper::Transport
 
       ciphertext = cipher.update(str) + cipher.final
 
-      "#@salt#{iv}#{ciphertext}"
+      [@salt, iv, ciphertext]
     end
-    def dec str
-      salt = str[0,8]
-      iv   = str[8,16]
-      str  = str[24..-1]
-      
+    def dec salt, iv, str
       cipher = OpenSSL::Cipher::AES256.new(:CBC)
       cipher.decrypt
       cipher.iv = iv
@@ -52,42 +49,62 @@ module EventShipper::Transport
     end
   end
 
+  # Takes a Protocol::Transmission object, encrypts it and turns it into 
+  # something Decrypt can read from the wire. (a string)
+  #
   class Encrypt
+    include EventShipper::Protocol
+
     def initialize user, password
       @user = user
-      @encryption = Encryption.new(password)
+      @algo = AES256.new(password)
     end
 
-    def call string
-      ciphertext = @encryption.enc string
-      "#{@user}/#{ciphertext}"
+    def en transmission
+      salt, iv, ciphertext = @algo.enc(transmission)
+
+      encrypted(
+        iv: iv, 
+        salt: salt, 
+        user: @user, 
+        ciphertext: ciphertext).serialize_to_string
     end
   end
 
   class Decrypt
-    def initialize userdb
-      @userdb = userdb.inject({}) do |hash, (key, value)|
-        hash[key] = Encryption.new(value)
-        hash
-      end
+    include EventShipper::Protocol
+    
+    def initialize user_db
+      @user_db = user_db.
+        inject({}) { |h, (user, pwd)| 
+          h[user] = AES256.new(pwd); h }
     end
 
-    def call str
-      user, _, ciphertext = str.partition('/')
+    def de encrypted_string
+      message = parse_encrypted(encrypted_string)
+
+      if algo = @user_db[message.user]
+        transmission = algo.dec(
+          message.salt, 
+          message.iv, 
+          message.ciphertext)
 
-      if encryption = @userdb[user]
-        encryption.dec(ciphertext)
+        return transmission
+      else
+        warn "No such user #{message.user} in database; cannot decrypt."
+        return nil
       end
     end
   end
+
 end
 
 if $0 == __FILE__
-  enc = EventShipper::Transport::Encryption.new 'password'
+  enc = EventShipper::Filter::Encryption.new 'password'
   str = enc.enc 'A very secret text'
   puts str
 
-  dec = EventShipper::Transport::Encryption.new 'password'
+  dec = EventShipper::Filter::Encryption.new 'password'
   puts dec.dec(str)
   exit
 
@@ -95,8 +112,8 @@ if $0 == __FILE__
   puts "Doing it a 1000 times"
   msg = "foobar"*20
   puts Benchmark.measure { 
-    e = EventShipper::Transport::Encrypt.new('test', 'password')
-    d = EventShipper::Transport::Decrypt.new('test' => 'password')
+    e = EventShipper::Filter::Encrypt.new('test', 'password')
+    d = EventShipper::Filter::Decrypt.new('test' => 'password')
     1000.times do d.call(e.call(msg)) end
   }
   

data/lib/event_shipper/filter/transmission.rb

@@ -0,0 +1,30 @@
+
+require 'yajl/json_gem'
+
+require 'event_shipper/protocol'
+
+module EventShipper::Filter
+  # Serializes a hash by wrapping it into a Protocol::Transmission object 
+  # and then serializing it to a string. 
+  #
+  class Transmission
+    include EventShipper::Protocol
+
+    def en hash
+      transmission(
+        event(
+          queue: 'queue', 
+          json: hash.to_json)).serialize_to_string
+    end
+    def de string
+      transmission = parse_transmission(string)
+
+      if transmission.version != 1
+        warn "Received a transmission with version #{transmission.version}, but this code parses version 1."
+      end
+
+      event = transmission.events.first
+      JSON.parse(event.json)
+    end
+  end
+end

data/lib/event_shipper/filters.rb

@@ -0,0 +1,5 @@
+
+module EventShipper::Filter; end
+
+require_relative 'filter/transmission'
+require_relative 'filter/encrypt'

data/lib/event_shipper/log_tailer.rb

@@ -18,19 +18,24 @@ module EventShipper
       @thread.abort_on_exception = true
     end
 
-    def thread_main
+    def thread_main once=false
       File.open(@path) do |log|
         log.seek 0, IO::SEEK_END
         log.extend File::Tail
 
+        log.max_interval = 5
+        log.interval = 1
+
         log.tail do |line|
           issue line
+
+          return if once
         end
       end
     end
     def issue line
       event = Event.new @host, @path, line
-      @transport.send "queue/#{event.to_json}"
+      @transport.send event.to_hash
     end
 
     def join

data/lib/event_shipper/protocol.rb

@@ -0,0 +1,26 @@
+
+require_relative 'protocol/v1.pb.rb'
+
+module EventShipper::Protocol
+  def transmission *events
+    Transmission.new(
+      version: 1, 
+      events: events)
+  end
+
+  def event attributes={}
+    Event.new(attributes)
+  end
+  def encrypted attributes={}
+    Encrypted.new(attributes)
+  end
+
+  def parse_encrypted string
+    Encrypted.new.tap { |o|
+      o.parse(string) }
+  end
+  def parse_transmission string
+    Transmission.new.tap { |o| 
+      o.parse(string) }
+  end
+end

data/lib/event_shipper/protocol/v1.pb.rb

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+# Generated by the protocol buffer compiler. DO NOT EDIT!
+
+require 'protocol_buffers'
+
+module EventShipper
+  module Protocol
+    # forward declarations
+    class Encrypted < ::ProtocolBuffers::Message; end
+    class Event < ::ProtocolBuffers::Message; end
+    class Transmission < ::ProtocolBuffers::Message; end
+
+    class Encrypted < ::ProtocolBuffers::Message
+      required :bytes, :salt, 1
+      required :bytes, :iv, 2
+      required :bytes, :ciphertext, 3
+      required :string, :user, 4
+    end
+
+    class Event < ::ProtocolBuffers::Message
+      required :string, :queue, 1
+      required :string, :json, 2
+    end
+
+    class Transmission < ::ProtocolBuffers::Message
+      required :uint32, :version, 1
+      repeated ::EventShipper::Protocol::Event, :events, 2
+    end
+
+  end
+end

data/lib/event_shipper/protocol/v1.proto

@@ -0,0 +1,30 @@
+
+// V1 of the on the wire protocol. Use
+//  rprotoc -o lib/event_shipper/protocol/ lib/event_shipper/protocol/v1.proto
+// to generate this. 
+
+package event_shipper.protocol;
+
+message Encrypted {
+  required bytes salt = 1; 
+  required bytes iv = 2; 
+  required bytes ciphertext = 3; 
+  required string user = 4; 
+}
+
+message Event {
+  // The queue to write this event to.
+  required string queue = 1; 
+  // A JSON hash of the event
+  required string json = 2; 
+}
+
+message Transmission {
+  // Should be set to the number 1 to match this file (v1.proto)
+  // Allows for breaking change versioning on top of field versioning. 
+  //
+  required uint32 version = 1; 
+
+  // Currently, a transmission will only include one event. This may change.
+  repeated Event events = 2; 
+}

data/lib/event_shipper/proxy.rb

@@ -3,11 +3,10 @@ require 'redis'
 require 'yaml'
 require 'clamp'
 
-module EventShipper
-
-  require_relative 'transport/encrypt'
-  require_relative 'transport/udp'
+require_relative 'filters'
+require_relative 'udp'
 
+module EventShipper
   class Proxy < Clamp::Command
 
     option %w(-c --config), "FILE", "Location of the configuration file."
@@ -20,13 +19,13 @@ module EventShipper
 
       listen = configuration['listen']
       host, port = listen['host'], listen['port']
-      transport = Transport::UDP.new(host, port)
+      transport = UDP.new(host, port)
 
       if configuration['encrypt']
         users = configuration['users']
         userdb = Hash[users.map { |str| str.split('/') }]
 
-        transport.filter = Transport::Decrypt.new(userdb)
+        transport.wrap Filter::Decrypt.new(userdb)
       end
 
       redis_config = configuration['redis']
@@ -35,7 +34,7 @@ module EventShipper
         port: redis_config['port'])
 
       transport.dispatch { |queue, message| 
-        # puts message
+        # puts message }
         redis.lpush queue, message }
     end
   end

data/lib/event_shipper/shipper.rb

@@ -2,11 +2,10 @@
 require 'yaml'
 require 'clamp'
 
-module EventShipper
+require_relative 'udp'
+require_relative 'log_tailer'
 
-  require 'event_shipper/transport/encrypt'
-  require 'event_shipper/transport/udp'
-  require 'event_shipper/log_tailer'
+module EventShipper
 
   class Shipper < Clamp::Command
     option %w(-c --config), "FILE", "Location of the configuration file."
@@ -18,11 +17,11 @@ module EventShipper
       configuration = YAML.load_file(config)
 
       host, port = configuration['target'].split(':')
-      transport = Transport::UDP.new(host, port)
+      transport = UDP.new(host, port)
 
       if configuration['encrypt'] 
         user, password = configuration.values_at('user', 'password')
-        transport.filter = Transport::Encrypt.new(user, password)
+        transport.wrap Filter::Encrypt.new(user, password)
       end
 
       tailers = configuration["logs"].map do |path|

data/lib/event_shipper/{transport/udp.rb → udp.rb}

@@ -1,17 +1,23 @@
 require 'socket'
 
-module EventShipper::Transport
+require_relative 'filters'
+
+module EventShipper
   class UDP
     def initialize host, port
       @host, @port = host, port
       @socket = UDPSocket.new
-      @filter = nil
-
+      
+      # The Bson-Filter acts as a terminator for hash based messages and
+      # turns things into BSON for the wire. 
+      @filters = []
+      wrap Filter::Transmission.new
+      
       @messages_per_period = 0
       @period_start = Time.now
       Thread.start do
         loop do
-          puts "total #{@messages_per_period / (Time.now - @period_start)} msgs/s"
+          puts "total #{(@messages_per_period / (Time.now - @period_start)).round(3)} msgs/s"
           
           @messages_per_period = 0
           @period_start = Time.now
@@ -21,19 +27,20 @@ module EventShipper::Transport
       end
     end
 
-    attr_writer :filter
+    def wrap filter
+      @filters << filter
+    end
 
-    def filter string
-      if @filter
-        @filter.call(string)
-      else
-        string
-      end
+    def encode obj
+      @filters.inject(obj) { |o, f| f.en(o) }
+    end
+    def decode obj
+      @filters.reverse.inject(obj) { |o, f| f.de(o) }
     end
 
-    def send string
+    def send hash
       @messages_per_period += 1
-      @socket.send filter(string), 
+      @socket.send encode(hash), 
         0,    # flags...
         @host, @port
     end
@@ -46,8 +53,9 @@ module EventShipper::Transport
         datagram, source_info = @socket.recvfrom(10 * 1024)
         @messages_per_period += 1
 
-        queue, _, message = filter(datagram).partition('/')
-        yield queue, message
+        hash = decode(datagram)
+
+        yield nil, hash
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: event-shipper
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-03 00:00:00.000000000 Z
+date: 2013-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -91,6 +91,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-protocol-buffers
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! "\n    event_shipper reads log files and sends each line in logstash
   JSON format \n    via encrypted UDP to redis.\n  "
 email: kaspar.schiess@absurd.li
@@ -104,13 +120,20 @@ files:
 - HISTORY
 - LICENSE
 - README
+- lib/event_shipper/benchmark.rb
 - lib/event_shipper/event.rb
+- lib/event_shipper/filter/encrypt.rb
+- lib/event_shipper/filter/transmission.rb
+- lib/event_shipper/filters.rb
 - lib/event_shipper/log_tailer.rb
+- lib/event_shipper/protocol/v1.pb.rb
+- lib/event_shipper/protocol/v1.proto
+- lib/event_shipper/protocol.rb
 - lib/event_shipper/proxy.rb
 - lib/event_shipper/shipper.rb
-- lib/event_shipper/transport/encrypt.rb
-- lib/event_shipper/transport/udp.rb
+- lib/event_shipper/udp.rb
 - lib/event_shipper.rb
+- bin/bench
 - bin/esproxy
 - bin/esshipper
 - bin/producer