event-shipper 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/HISTORY CHANGED
@@ -1,4 +1,10 @@
1
1
 
2
+ = 0.2
3
+
4
+ + Proper framing of transmissions; transmissions are now versioned and
5
+ should work even in environments where server and client are not the same
6
+ gem. This is the case for versions >= 0.2.
7
+
2
8
  = 0.1
3
9
 
4
10
  * First version.
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Benchmarks message code.
4
+
5
+ require 'bundler/setup'
6
+
7
+ $:.unshift File.dirname(__FILE__) + "/../lib"
8
+ require 'event_shipper'
9
+ require 'event_shipper/benchmark'
10
+
11
+ EventShipper::Benchmark.run
@@ -6,6 +6,6 @@
6
6
  require 'bundler/setup'
7
7
 
8
8
  $:.unshift File.dirname(__FILE__) + "/../lib"
9
- require 'event_shipper/proxy'
9
+ require 'event_shipper'
10
10
 
11
11
  EventShipper::Proxy.run
@@ -6,6 +6,6 @@
6
6
  require 'bundler/setup'
7
7
 
8
8
  $:.unshift File.dirname(__FILE__) + "/../lib"
9
- require 'event_shipper/shipper'
9
+ require 'event_shipper'
10
10
 
11
11
  EventShipper::Shipper.run
@@ -0,0 +1,6 @@
1
+
2
+
3
+ module EventShipper; end
4
+
5
+ require 'event_shipper/proxy'
6
+ require 'event_shipper/shipper'
@@ -0,0 +1,46 @@
1
+
2
+ require 'benchmark'
3
+ require 'clamp'
4
+
5
+ module EventShipper
6
+ class Benchmark < Clamp::Command
7
+ def initialize *args
8
+ super
9
+
10
+ @s = UDP.new('localhost', 5050)
11
+ @s.wrap Filter::Encrypt.new('user', 'password')
12
+
13
+ @r = UDP.new('localhost', 5050)
14
+ @r.wrap Filter::Decrypt.new('user' => 'password')
15
+ end
16
+
17
+ def execute
18
+ puts "Simulates 1000 times the sending of a simple message: "
19
+ measure do
20
+ 1000.times do _send_message end
21
+ end
22
+
23
+ puts "Simulates 1000 times the receiving of a message: "
24
+ msg = _send_message
25
+ measure do
26
+ 1000.times do _receive_message(msg) end
27
+ end
28
+ end
29
+
30
+ def measure
31
+ puts ::Benchmark.measure {
32
+ yield
33
+ }
34
+ end
35
+
36
+ def _send_message
37
+ @s.encode(
38
+ source: 'benchmark',
39
+ event: 'this is the event')
40
+ end
41
+
42
+ def _receive_message msg
43
+ @r.decode(msg)
44
+ end
45
+ end
46
+ end
@@ -1,5 +1,4 @@
1
1
 
2
- require 'yajl'
3
2
  require 'addressable/uri'
4
3
 
5
4
  module EventShipper
@@ -29,13 +28,5 @@ module EventShipper
29
28
  '@message' => @message
30
29
  }
31
30
  end
32
-
33
- def to_json
34
- Yajl::Encoder.encode(to_hash)
35
- end
36
-
37
- def to_s
38
- to_json
39
- end
40
31
  end
41
32
  end
@@ -1,9 +1,10 @@
1
1
 
2
2
  require 'openssl'
3
3
 
4
- module EventShipper; end
5
- module EventShipper::Transport
6
- class Encryption
4
+ require 'event_shipper/protocol'
5
+
6
+ module EventShipper::Filter
7
+ class AES256
7
8
  def initialize password
8
9
  @password = password
9
10
  @salt = generate_salt
@@ -36,13 +37,9 @@ module EventShipper::Transport
36
37
 
37
38
  ciphertext = cipher.update(str) + cipher.final
38
39
 
39
- "#@salt#{iv}#{ciphertext}"
40
+ [@salt, iv, ciphertext]
40
41
  end
41
- def dec str
42
- salt = str[0,8]
43
- iv = str[8,16]
44
- str = str[24..-1]
45
-
42
+ def dec salt, iv, str
46
43
  cipher = OpenSSL::Cipher::AES256.new(:CBC)
47
44
  cipher.decrypt
48
45
  cipher.iv = iv
@@ -52,42 +49,62 @@ module EventShipper::Transport
52
49
  end
53
50
  end
54
51
 
52
+ # Takes a Protocol::Transmission object, encrypts it and turns it into
53
+ # something Decrypt can read from the wire. (a string)
54
+ #
55
55
  class Encrypt
56
+ include EventShipper::Protocol
57
+
56
58
  def initialize user, password
57
59
  @user = user
58
- @encryption = Encryption.new(password)
60
+ @algo = AES256.new(password)
59
61
  end
60
62
 
61
- def call string
62
- ciphertext = @encryption.enc string
63
- "#{@user}/#{ciphertext}"
63
+ def en transmission
64
+ salt, iv, ciphertext = @algo.enc(transmission)
65
+
66
+ encrypted(
67
+ iv: iv,
68
+ salt: salt,
69
+ user: @user,
70
+ ciphertext: ciphertext).serialize_to_string
64
71
  end
65
72
  end
66
73
 
67
74
  class Decrypt
68
- def initialize userdb
69
- @userdb = userdb.inject({}) do |hash, (key, value)|
70
- hash[key] = Encryption.new(value)
71
- hash
72
- end
75
+ include EventShipper::Protocol
76
+
77
+ def initialize user_db
78
+ @user_db = user_db.
79
+ inject({}) { |h, (user, pwd)|
80
+ h[user] = AES256.new(pwd); h }
73
81
  end
74
82
 
75
- def call str
76
- user, _, ciphertext = str.partition('/')
83
+ def de encrypted_string
84
+ message = parse_encrypted(encrypted_string)
85
+
86
+ if algo = @user_db[message.user]
87
+ transmission = algo.dec(
88
+ message.salt,
89
+ message.iv,
90
+ message.ciphertext)
77
91
 
78
- if encryption = @userdb[user]
79
- encryption.dec(ciphertext)
92
+ return transmission
93
+ else
94
+ warn "No such user #{message.user} in database; cannot decrypt."
95
+ return nil
80
96
  end
81
97
  end
82
98
  end
99
+
83
100
  end
84
101
 
85
102
  if $0 == __FILE__
86
- enc = EventShipper::Transport::Encryption.new 'password'
103
+ enc = EventShipper::Filter::Encryption.new 'password'
87
104
  str = enc.enc 'A very secret text'
88
105
  puts str
89
106
 
90
- dec = EventShipper::Transport::Encryption.new 'password'
107
+ dec = EventShipper::Filter::Encryption.new 'password'
91
108
  puts dec.dec(str)
92
109
  exit
93
110
 
@@ -95,8 +112,8 @@ if $0 == __FILE__
95
112
  puts "Doing it a 1000 times"
96
113
  msg = "foobar"*20
97
114
  puts Benchmark.measure {
98
- e = EventShipper::Transport::Encrypt.new('test', 'password')
99
- d = EventShipper::Transport::Decrypt.new('test' => 'password')
115
+ e = EventShipper::Filter::Encrypt.new('test', 'password')
116
+ d = EventShipper::Filter::Decrypt.new('test' => 'password')
100
117
  1000.times do d.call(e.call(msg)) end
101
118
  }
102
119
 
@@ -0,0 +1,30 @@
1
+
2
+ require 'yajl/json_gem'
3
+
4
+ require 'event_shipper/protocol'
5
+
6
+ module EventShipper::Filter
7
+ # Serializes a hash by wrapping it into a Protocol::Transmission object
8
+ # and then serializing it to a string.
9
+ #
10
+ class Transmission
11
+ include EventShipper::Protocol
12
+
13
+ def en hash
14
+ transmission(
15
+ event(
16
+ queue: 'queue',
17
+ json: hash.to_json)).serialize_to_string
18
+ end
19
+ def de string
20
+ transmission = parse_transmission(string)
21
+
22
+ if transmission.version != 1
23
+ warn "Received a transmission with version #{transmission.version}, but this code parses version 1."
24
+ end
25
+
26
+ event = transmission.events.first
27
+ JSON.parse(event.json)
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,5 @@
1
+
2
+ module EventShipper::Filter; end
3
+
4
+ require_relative 'filter/transmission'
5
+ require_relative 'filter/encrypt'
@@ -18,19 +18,24 @@ module EventShipper
18
18
  @thread.abort_on_exception = true
19
19
  end
20
20
 
21
- def thread_main
21
+ def thread_main once=false
22
22
  File.open(@path) do |log|
23
23
  log.seek 0, IO::SEEK_END
24
24
  log.extend File::Tail
25
25
 
26
+ log.max_interval = 5
27
+ log.interval = 1
28
+
26
29
  log.tail do |line|
27
30
  issue line
31
+
32
+ return if once
28
33
  end
29
34
  end
30
35
  end
31
36
  def issue line
32
37
  event = Event.new @host, @path, line
33
- @transport.send "queue/#{event.to_json}"
38
+ @transport.send event.to_hash
34
39
  end
35
40
 
36
41
  def join
@@ -0,0 +1,26 @@
1
+
2
+ require_relative 'protocol/v1.pb.rb'
3
+
4
+ module EventShipper::Protocol
5
+ def transmission *events
6
+ Transmission.new(
7
+ version: 1,
8
+ events: events)
9
+ end
10
+
11
+ def event attributes={}
12
+ Event.new(attributes)
13
+ end
14
+ def encrypted attributes={}
15
+ Encrypted.new(attributes)
16
+ end
17
+
18
+ def parse_encrypted string
19
+ Encrypted.new.tap { |o|
20
+ o.parse(string) }
21
+ end
22
+ def parse_transmission string
23
+ Transmission.new.tap { |o|
24
+ o.parse(string) }
25
+ end
26
+ end
@@ -0,0 +1,31 @@
1
+ #!/usr/bin/env ruby
2
+ # Generated by the protocol buffer compiler. DO NOT EDIT!
3
+
4
+ require 'protocol_buffers'
5
+
6
+ module EventShipper
7
+ module Protocol
8
+ # forward declarations
9
+ class Encrypted < ::ProtocolBuffers::Message; end
10
+ class Event < ::ProtocolBuffers::Message; end
11
+ class Transmission < ::ProtocolBuffers::Message; end
12
+
13
+ class Encrypted < ::ProtocolBuffers::Message
14
+ required :bytes, :salt, 1
15
+ required :bytes, :iv, 2
16
+ required :bytes, :ciphertext, 3
17
+ required :string, :user, 4
18
+ end
19
+
20
+ class Event < ::ProtocolBuffers::Message
21
+ required :string, :queue, 1
22
+ required :string, :json, 2
23
+ end
24
+
25
+ class Transmission < ::ProtocolBuffers::Message
26
+ required :uint32, :version, 1
27
+ repeated ::EventShipper::Protocol::Event, :events, 2
28
+ end
29
+
30
+ end
31
+ end
@@ -0,0 +1,30 @@
1
+
2
+ // V1 of the on the wire protocol. Use
3
+ // rprotoc -o lib/event_shipper/protocol/ lib/event_shipper/protocol/v1.proto
4
+ // to generate this.
5
+
6
+ package event_shipper.protocol;
7
+
8
+ message Encrypted {
9
+ required bytes salt = 1;
10
+ required bytes iv = 2;
11
+ required bytes ciphertext = 3;
12
+ required string user = 4;
13
+ }
14
+
15
+ message Event {
16
+ // The queue to write this event to.
17
+ required string queue = 1;
18
+ // A JSON hash of the event
19
+ required string json = 2;
20
+ }
21
+
22
+ message Transmission {
23
+ // Should be set to the number 1 to match this file (v1.proto)
24
+ // Allows for breaking change versioning on top of field versioning.
25
+ //
26
+ required uint32 version = 1;
27
+
28
+ // Currently, a transmission will only include one event. This may change.
29
+ repeated Event events = 2;
30
+ }
@@ -3,11 +3,10 @@ require 'redis'
3
3
  require 'yaml'
4
4
  require 'clamp'
5
5
 
6
- module EventShipper
7
-
8
- require_relative 'transport/encrypt'
9
- require_relative 'transport/udp'
6
+ require_relative 'filters'
7
+ require_relative 'udp'
10
8
 
9
+ module EventShipper
11
10
  class Proxy < Clamp::Command
12
11
 
13
12
  option %w(-c --config), "FILE", "Location of the configuration file."
@@ -20,13 +19,13 @@ module EventShipper
20
19
 
21
20
  listen = configuration['listen']
22
21
  host, port = listen['host'], listen['port']
23
- transport = Transport::UDP.new(host, port)
22
+ transport = UDP.new(host, port)
24
23
 
25
24
  if configuration['encrypt']
26
25
  users = configuration['users']
27
26
  userdb = Hash[users.map { |str| str.split('/') }]
28
27
 
29
- transport.filter = Transport::Decrypt.new(userdb)
28
+ transport.wrap Filter::Decrypt.new(userdb)
30
29
  end
31
30
 
32
31
  redis_config = configuration['redis']
@@ -35,7 +34,7 @@ module EventShipper
35
34
  port: redis_config['port'])
36
35
 
37
36
  transport.dispatch { |queue, message|
38
- # puts message
37
+ # puts message }
39
38
  redis.lpush queue, message }
40
39
  end
41
40
  end
@@ -2,11 +2,10 @@
2
2
  require 'yaml'
3
3
  require 'clamp'
4
4
 
5
- module EventShipper
5
+ require_relative 'udp'
6
+ require_relative 'log_tailer'
6
7
 
7
- require 'event_shipper/transport/encrypt'
8
- require 'event_shipper/transport/udp'
9
- require 'event_shipper/log_tailer'
8
+ module EventShipper
10
9
 
11
10
  class Shipper < Clamp::Command
12
11
  option %w(-c --config), "FILE", "Location of the configuration file."
@@ -18,11 +17,11 @@ module EventShipper
18
17
  configuration = YAML.load_file(config)
19
18
 
20
19
  host, port = configuration['target'].split(':')
21
- transport = Transport::UDP.new(host, port)
20
+ transport = UDP.new(host, port)
22
21
 
23
22
  if configuration['encrypt']
24
23
  user, password = configuration.values_at('user', 'password')
25
- transport.filter = Transport::Encrypt.new(user, password)
24
+ transport.wrap Filter::Encrypt.new(user, password)
26
25
  end
27
26
 
28
27
  tailers = configuration["logs"].map do |path|
@@ -1,17 +1,23 @@
1
1
  require 'socket'
2
2
 
3
- module EventShipper::Transport
3
+ require_relative 'filters'
4
+
5
+ module EventShipper
4
6
  class UDP
5
7
  def initialize host, port
6
8
  @host, @port = host, port
7
9
  @socket = UDPSocket.new
8
- @filter = nil
9
-
10
+
11
+ # The Bson-Filter acts as a terminator for hash based messages and
12
+ # turns things into BSON for the wire.
13
+ @filters = []
14
+ wrap Filter::Transmission.new
15
+
10
16
  @messages_per_period = 0
11
17
  @period_start = Time.now
12
18
  Thread.start do
13
19
  loop do
14
- puts "total #{@messages_per_period / (Time.now - @period_start)} msgs/s"
20
+ puts "total #{(@messages_per_period / (Time.now - @period_start)).round(3)} msgs/s"
15
21
 
16
22
  @messages_per_period = 0
17
23
  @period_start = Time.now
@@ -21,19 +27,20 @@ module EventShipper::Transport
21
27
  end
22
28
  end
23
29
 
24
- attr_writer :filter
30
+ def wrap filter
31
+ @filters << filter
32
+ end
25
33
 
26
- def filter string
27
- if @filter
28
- @filter.call(string)
29
- else
30
- string
31
- end
34
+ def encode obj
35
+ @filters.inject(obj) { |o, f| f.en(o) }
36
+ end
37
+ def decode obj
38
+ @filters.reverse.inject(obj) { |o, f| f.de(o) }
32
39
  end
33
40
 
34
- def send string
41
+ def send hash
35
42
  @messages_per_period += 1
36
- @socket.send filter(string),
43
+ @socket.send encode(hash),
37
44
  0, # flags...
38
45
  @host, @port
39
46
  end
@@ -46,8 +53,9 @@ module EventShipper::Transport
46
53
  datagram, source_info = @socket.recvfrom(10 * 1024)
47
54
  @messages_per_period += 1
48
55
 
49
- queue, _, message = filter(datagram).partition('/')
50
- yield queue, message
56
+ hash = decode(datagram)
57
+
58
+ yield nil, hash
51
59
  end
52
60
  end
53
61
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: event-shipper
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.2.0
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-05-03 00:00:00.000000000 Z
12
+ date: 2013-05-14 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: redis
@@ -91,6 +91,22 @@ dependencies:
91
91
  - - ! '>='
92
92
  - !ruby/object:Gem::Version
93
93
  version: '0'
94
+ - !ruby/object:Gem::Dependency
95
+ name: ruby-protocol-buffers
96
+ requirement: !ruby/object:Gem::Requirement
97
+ none: false
98
+ requirements:
99
+ - - ! '>='
100
+ - !ruby/object:Gem::Version
101
+ version: '0'
102
+ type: :runtime
103
+ prerelease: false
104
+ version_requirements: !ruby/object:Gem::Requirement
105
+ none: false
106
+ requirements:
107
+ - - ! '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
94
110
  description: ! "\n event_shipper reads log files and sends each line in logstash
95
111
  JSON format \n via encrypted UDP to redis.\n "
96
112
  email: kaspar.schiess@absurd.li
@@ -104,13 +120,20 @@ files:
104
120
  - HISTORY
105
121
  - LICENSE
106
122
  - README
123
+ - lib/event_shipper/benchmark.rb
107
124
  - lib/event_shipper/event.rb
125
+ - lib/event_shipper/filter/encrypt.rb
126
+ - lib/event_shipper/filter/transmission.rb
127
+ - lib/event_shipper/filters.rb
108
128
  - lib/event_shipper/log_tailer.rb
129
+ - lib/event_shipper/protocol/v1.pb.rb
130
+ - lib/event_shipper/protocol/v1.proto
131
+ - lib/event_shipper/protocol.rb
109
132
  - lib/event_shipper/proxy.rb
110
133
  - lib/event_shipper/shipper.rb
111
- - lib/event_shipper/transport/encrypt.rb
112
- - lib/event_shipper/transport/udp.rb
134
+ - lib/event_shipper/udp.rb
113
135
  - lib/event_shipper.rb
136
+ - bin/bench
114
137
  - bin/esproxy
115
138
  - bin/esshipper
116
139
  - bin/producer