event-shipper 0.1.0

data/HISTORY

@@ -0,0 +1,4 @@
+
+= 0.1
+
+  * First version.

data/LICENSE

@@ -0,0 +1,23 @@
+
+ Copyright (c) 2013 Kaspar Schiess
+
+ Permission is hereby granted, free of charge, to any person
+ obtaining a copy of this software and associated documentation
+ files (the "Software"), to deal in the Software without
+ restriction, including without limitation the rights to use,
+ copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the
+ Software is furnished to do so, subject to the following
+ conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,57 @@
+
+event_shipper reads log files and sends each line in logstash JSON format 
+via encrypted UDP to redis. 
+
+LINKS
+
+logstash is awesome and here: 
+  http://logstash.net/
+
+this uses a CA and public/private key architecture: 
+  https://github.com/jordansissel/lumberjack
+
+and finally, here's woodchuck, which inspired this work: 
+  https://github.com/danryan/woodchuck
+
+SYNOPSIS
+
+To run a local example installation, start redis. Then: 
+
+  # Produce one log entry per second in the file 'test.log'
+  $ ./bin/producer 1
+
+  # Ship the log entries to localhost
+  $ ./bin/esshipper -c configs/test.yaml
+
+  # Receive entries and store in redis
+  $ ./bin/esproxy -c configs/proxy.yaml
+
+PERFORMANCE
+
+Good. 
+
+ENCRYPTION
+
+Packets are encrypted using a simple AES256 symmetric encryption with a 
+preshared user/password tuple on the server and the client. This should be 
+good to keep people from reading your logs, 
+
+If someone with deep cryptographic knowledge wants to criticise my use of 
+algorithms, please, you're welcome! Just keep in mind that I've selected the
+current method to be simple to set up. 
+
+CONTRIBUTE
+
+This gem is hosted on bitbucket at 
+  https://bitbucket.org/kschiess/event_shipper
+
+Please file any issue you might have with the program there. Contributions
+are welcome - you might shoot me an email before starting, just to check if 
+the direction you take is welcome in the code base. 
+
+LICENSE
+
+This piece of software is under a MIT license, which means you can do pretty 
+much anything you want with it. Read the license, it's in the LICENSE file. 
+
+(c) 2013 Kaspar Schiess, Technology Astronauts

data/bin/esproxy

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# Tails one or more log files and ships logs as events over the network
+# via a custom encrypted protocol. Transport is UDP. 
+
+require 'bundler/setup'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'event_shipper/proxy'
+
+EventShipper::Proxy.run

data/bin/esshipper

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+# Tails one or more log files and ships logs as events over the network
+# via a custom encrypted protocol. Transport is UDP. 
+
+require 'bundler/setup'
+
+$:.unshift File.dirname(__FILE__) + "/../lib"
+require 'event_shipper/shipper'
+
+EventShipper::Shipper.run

data/bin/producer

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+# A trivial test script that produces a file called test.log and appends N 
+# messages per second to the file. 
+
+n = Integer(ARGV.first || 1)
+
+sleep_seconds = (1 / Float(n))
+
+File.open('test.log', 'a+') do |file|
+  file.sync = true
+
+  loop do
+    file.puts "A message (#{Time.now})"
+    sleep sleep_seconds
+  end
+end

data/configs/proxy.yaml

@@ -0,0 +1,12 @@
+
+encrypt: true
+users: 
+  - test/MyPassword
+
+listen: 
+  host: 127.0.0.1
+  port: 5050
+
+redis:
+  host: localhost
+  port: 6379

data/configs/test.yaml

@@ -0,0 +1,10 @@
+
+encrypt: true
+user: test
+password: MyPassword
+
+target: localhost:5050
+
+logs: 
+  - ./test.log
+

data/lib/event_shipper/event.rb

@@ -0,0 +1,41 @@
+
+require 'yajl'
+require 'addressable/uri'
+
+module EventShipper
+  class Event
+    def initialize host, path, line
+      @path = path
+      @line = line
+      @host = host
+      @timestamp = Time.now.utc.iso8601(6)
+      @source = Addressable::URI.new(
+        :scheme => 'file', 
+        :host => host, :path => path)
+      @message = line.chomp.strip
+      @fields = {}
+      @tags = []
+    end
+    
+    def to_hash
+      {
+        '@source' => @source.to_s,
+        '@type' => @source.scheme,
+        '@tags' => @tags,
+        '@fields' => @fields,
+        '@timestamp' => @timestamp,
+        '@source_host' => @host,
+        '@source_path' => @path,
+        '@message' => @message
+      }
+    end
+    
+    def to_json
+      Yajl::Encoder.encode(to_hash)
+    end
+    
+    def to_s
+      to_json
+    end
+  end
+end

data/lib/event_shipper/log_tailer.rb

@@ -0,0 +1,40 @@
+
+require 'time'
+require 'socket'
+require 'file-tail'
+
+require 'event_shipper/event'
+
+module EventShipper
+  class LogTailer
+    def initialize path, transport
+      @path = path
+      @transport = transport
+      @host = Socket.gethostname
+    end
+
+    def start
+      @thread = Thread.new(&method(:thread_main))
+      @thread.abort_on_exception = true
+    end
+
+    def thread_main
+      File.open(@path) do |log|
+        log.seek 0, IO::SEEK_END
+        log.extend File::Tail
+
+        log.tail do |line|
+          issue line
+        end
+      end
+    end
+    def issue line
+      event = Event.new @host, @path, line
+      @transport.send "queue/#{event.to_json}"
+    end
+
+    def join
+      @thread.join
+    end
+  end
+end

data/lib/event_shipper/proxy.rb

@@ -0,0 +1,42 @@
+
+require 'redis'
+require 'yaml'
+require 'clamp'
+
+module EventShipper
+
+  require_relative 'transport/encrypt'
+  require_relative 'transport/udp'
+
+  class Proxy < Clamp::Command
+
+    option %w(-c --config), "FILE", "Location of the configuration file."
+
+    def execute
+      fail "Please specify a configuration file to use on the command line. (--config)" \
+        unless config
+
+      configuration = YAML.load_file(config)
+
+      listen = configuration['listen']
+      host, port = listen['host'], listen['port']
+      transport = Transport::UDP.new(host, port)
+
+      if configuration['encrypt']
+        users = configuration['users']
+        userdb = Hash[users.map { |str| str.split('/') }]
+
+        transport.filter = Transport::Decrypt.new(userdb)
+      end
+
+      redis_config = configuration['redis']
+      redis = Redis.new(
+        host: redis_config['host'], 
+        port: redis_config['port'])
+
+      transport.dispatch { |queue, message| 
+        # puts message
+        redis.lpush queue, message }
+    end
+  end
+end

data/lib/event_shipper/shipper.rb

@@ -0,0 +1,41 @@
+
+require 'yaml'
+require 'clamp'
+
+module EventShipper
+
+  require 'event_shipper/transport/encrypt'
+  require 'event_shipper/transport/udp'
+  require 'event_shipper/log_tailer'
+
+  class Shipper < Clamp::Command
+    option %w(-c --config), "FILE", "Location of the configuration file."
+
+    def execute
+      fail "Please specify a configuration file to use on the command line. (--config)" \
+        unless config
+
+      configuration = YAML.load_file(config)
+
+      host, port = configuration['target'].split(':')
+      transport = Transport::UDP.new(host, port)
+
+      if configuration['encrypt'] 
+        user, password = configuration.values_at('user', 'password')
+        transport.filter = Transport::Encrypt.new(user, password)
+      end
+
+      tailers = configuration["logs"].map do |path|
+        tailer = LogTailer.new(path, transport)
+        tailer.start
+
+        tailer
+      end
+
+      # Block until everyone exits.
+      tailers.each do |tailer|
+        tailer.join
+      end
+    end
+  end
+end

data/lib/event_shipper/transport/encrypt.rb

@@ -0,0 +1,101 @@
+
+require 'openssl'
+
+module EventShipper; end
+module EventShipper::Transport
+  class Encryption
+    def initialize password
+      @password = password
+      @salt = generate_salt
+
+      @key = generate_key(password, @salt)
+    end
+
+    def generate_salt
+      OpenSSL::Random.random_bytes(8)
+    end
+
+    def generate_key password, salt
+      if [password, salt] == @last_key_seed
+        return @last_key 
+      end
+
+      iterations = 10000
+      key_length = 32
+
+      @last_key_seed = [password, salt]
+      @last_key = OpenSSL::PKCS5.pbkdf2_hmac_sha1(
+        password, salt, iterations, key_length)
+    end
+
+    def enc str
+      cipher = OpenSSL::Cipher::AES256.new(:CBC)
+      cipher.encrypt
+      # cipher.random_iv
+      cipher.key = @key
+
+      ciphertext = cipher.update(str) + cipher.final
+
+      "Salted__#{@salt}#{ciphertext}" #OpenSSL compatible
+    end
+    def dec str
+      salt = str[8..15]
+      str = str[16..-1]
+
+      cipher = OpenSSL::Cipher::AES256.new(:CBC)
+      cipher.decrypt
+      # cipher.random_iv
+      cipher.key = generate_key(@password, salt)
+
+      cipher.update(str) + cipher.final
+    end
+  end
+
+  class Encrypt
+    def initialize user, password
+      @user = user
+      @encryption = Encryption.new(password)
+    end
+
+    def call string
+      ciphertext = @encryption.enc string
+      "#{@user}/#{ciphertext}"
+    end
+  end
+
+  class Decrypt
+    def initialize userdb
+      @userdb = userdb.inject({}) do |hash, (key, value)|
+        hash[key] = Encryption.new(value)
+        hash
+      end
+    end
+
+    def call str
+      user, _, ciphertext = str.partition('/')
+
+      if encryption = @userdb[user]
+        encryption.dec(ciphertext)
+      end
+    end
+  end
+end
+
+if $0 == __FILE__
+  enc = EventShipper::Transport::Encryption.new 'password'
+  str = enc.enc 'A very secret text'
+  puts str
+
+  dec = EventShipper::Transport::Encryption.new 'password'
+  puts dec.dec(str)
+
+  require 'benchmark'
+  puts "Doing it a 1000 times"
+  msg = "foobar"*20
+  puts Benchmark.measure { 
+    e = EventShipper::Transport::Encrypt.new('test', 'password')
+    d = EventShipper::Transport::Decrypt.new('test' => 'password')
+    1000.times do d.call(e.call(msg)) end
+  }
+  
+end

data/lib/event_shipper/transport/udp.rb

@@ -0,0 +1,54 @@
+require 'socket'
+
+module EventShipper::Transport
+  class UDP
+    def initialize host, port
+      @host, @port = host, port
+      @socket = UDPSocket.new
+      @filter = nil
+
+      @messages_per_period = 0
+      @period_start = Time.now
+      Thread.start do
+        loop do
+          puts "total #{@messages_per_period / (Time.now - @period_start)} msgs/s"
+          
+          @messages_per_period = 0
+          @period_start = Time.now
+
+          sleep 10
+        end
+      end
+    end
+
+    attr_writer :filter
+
+    def filter string
+      if @filter
+        @filter.call(string)
+      else
+        string
+      end
+    end
+
+    def send string
+      @messages_per_period += 1
+      @socket.send filter(string), 
+        0,    # flags...
+        @host, @port
+    end
+
+    # Enters a loop, receiving messages, yielding them to the block. 
+    #
+    def dispatch
+      @socket.bind @host, @port
+      loop do
+        datagram, source_info = @socket.recvfrom(10 * 1024)
+        @messages_per_period += 1
+
+        queue, _, message = filter(datagram).partition('/')
+        yield queue, message
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,146 @@
+--- !ruby/object:Gem::Specification
+name: event-shipper
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Kaspar Schiess
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: clamp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: file-tail
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ! "\n    event_shipper reads log files and sends each line in logstash
+  JSON format \n    via encrypted UDP to redis.\n  "
+email: kaspar.schiess@absurd.li
+executables:
+- esproxy
+- esshipper
+extensions: []
+extra_rdoc_files:
+- README
+files:
+- HISTORY
+- LICENSE
+- README
+- lib/event_shipper/event.rb
+- lib/event_shipper/log_tailer.rb
+- lib/event_shipper/proxy.rb
+- lib/event_shipper/shipper.rb
+- lib/event_shipper/transport/encrypt.rb
+- lib/event_shipper/transport/udp.rb
+- lib/event_shipper.rb
+- bin/esproxy
+- bin/esshipper
+- bin/producer
+- configs/proxy.yaml
+- configs/test.yaml
+homepage: https://bitbucket.org/kschiess/event_shipper
+licenses: []
+post_install_message: 
+rdoc_options:
+- --main
+- README
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Ships log events to logstash.
+test_files: []
+has_rdoc: