event-shipper 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY +6 -0
- data/bin/bench +11 -0
- data/bin/esproxy +1 -1
- data/bin/esshipper +1 -1
- data/lib/event_shipper.rb +6 -0
- data/lib/event_shipper/benchmark.rb +46 -0
- data/lib/event_shipper/event.rb +0 -9
- data/lib/event_shipper/{transport → filter}/encrypt.rb +43 -26
- data/lib/event_shipper/filter/transmission.rb +30 -0
- data/lib/event_shipper/filters.rb +5 -0
- data/lib/event_shipper/log_tailer.rb +7 -2
- data/lib/event_shipper/protocol.rb +26 -0
- data/lib/event_shipper/protocol/v1.pb.rb +31 -0
- data/lib/event_shipper/protocol/v1.proto +30 -0
- data/lib/event_shipper/proxy.rb +6 -7
- data/lib/event_shipper/shipper.rb +5 -6
- data/lib/event_shipper/{transport/udp.rb → udp.rb} +23 -15
- metadata +27 -4
data/HISTORY
CHANGED
data/bin/bench
ADDED
data/bin/esproxy
CHANGED
data/bin/esshipper
CHANGED
data/lib/event_shipper.rb
CHANGED
@@ -0,0 +1,46 @@
|
|
1
|
+
|
2
|
+
require 'benchmark'
|
3
|
+
require 'clamp'
|
4
|
+
|
5
|
+
module EventShipper
|
6
|
+
class Benchmark < Clamp::Command
|
7
|
+
def initialize *args
|
8
|
+
super
|
9
|
+
|
10
|
+
@s = UDP.new('localhost', 5050)
|
11
|
+
@s.wrap Filter::Encrypt.new('user', 'password')
|
12
|
+
|
13
|
+
@r = UDP.new('localhost', 5050)
|
14
|
+
@r.wrap Filter::Decrypt.new('user' => 'password')
|
15
|
+
end
|
16
|
+
|
17
|
+
def execute
|
18
|
+
puts "Simulates 1000 times the sending of a simple message: "
|
19
|
+
measure do
|
20
|
+
1000.times do _send_message end
|
21
|
+
end
|
22
|
+
|
23
|
+
puts "Simulates 1000 times the receiving of a message: "
|
24
|
+
msg = _send_message
|
25
|
+
measure do
|
26
|
+
1000.times do _receive_message(msg) end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def measure
|
31
|
+
puts ::Benchmark.measure {
|
32
|
+
yield
|
33
|
+
}
|
34
|
+
end
|
35
|
+
|
36
|
+
def _send_message
|
37
|
+
@s.encode(
|
38
|
+
source: 'benchmark',
|
39
|
+
event: 'this is the event')
|
40
|
+
end
|
41
|
+
|
42
|
+
def _receive_message msg
|
43
|
+
@r.decode(msg)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
data/lib/event_shipper/event.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
|
2
|
-
require 'yajl'
|
3
2
|
require 'addressable/uri'
|
4
3
|
|
5
4
|
module EventShipper
|
@@ -29,13 +28,5 @@ module EventShipper
|
|
29
28
|
'@message' => @message
|
30
29
|
}
|
31
30
|
end
|
32
|
-
|
33
|
-
def to_json
|
34
|
-
Yajl::Encoder.encode(to_hash)
|
35
|
-
end
|
36
|
-
|
37
|
-
def to_s
|
38
|
-
to_json
|
39
|
-
end
|
40
31
|
end
|
41
32
|
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
|
2
2
|
require 'openssl'
|
3
3
|
|
4
|
-
|
5
|
-
|
6
|
-
|
4
|
+
require 'event_shipper/protocol'
|
5
|
+
|
6
|
+
module EventShipper::Filter
|
7
|
+
class AES256
|
7
8
|
def initialize password
|
8
9
|
@password = password
|
9
10
|
@salt = generate_salt
|
@@ -36,13 +37,9 @@ module EventShipper::Transport
|
|
36
37
|
|
37
38
|
ciphertext = cipher.update(str) + cipher.final
|
38
39
|
|
39
|
-
|
40
|
+
[@salt, iv, ciphertext]
|
40
41
|
end
|
41
|
-
def dec str
|
42
|
-
salt = str[0,8]
|
43
|
-
iv = str[8,16]
|
44
|
-
str = str[24..-1]
|
45
|
-
|
42
|
+
def dec salt, iv, str
|
46
43
|
cipher = OpenSSL::Cipher::AES256.new(:CBC)
|
47
44
|
cipher.decrypt
|
48
45
|
cipher.iv = iv
|
@@ -52,42 +49,62 @@ module EventShipper::Transport
|
|
52
49
|
end
|
53
50
|
end
|
54
51
|
|
52
|
+
# Takes a Protocol::Transmission object, encrypts it and turns it into
|
53
|
+
# something Decrypt can read from the wire. (a string)
|
54
|
+
#
|
55
55
|
class Encrypt
|
56
|
+
include EventShipper::Protocol
|
57
|
+
|
56
58
|
def initialize user, password
|
57
59
|
@user = user
|
58
|
-
@
|
60
|
+
@algo = AES256.new(password)
|
59
61
|
end
|
60
62
|
|
61
|
-
def
|
62
|
-
ciphertext = @
|
63
|
-
|
63
|
+
def en transmission
|
64
|
+
salt, iv, ciphertext = @algo.enc(transmission)
|
65
|
+
|
66
|
+
encrypted(
|
67
|
+
iv: iv,
|
68
|
+
salt: salt,
|
69
|
+
user: @user,
|
70
|
+
ciphertext: ciphertext).serialize_to_string
|
64
71
|
end
|
65
72
|
end
|
66
73
|
|
67
74
|
class Decrypt
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
75
|
+
include EventShipper::Protocol
|
76
|
+
|
77
|
+
def initialize user_db
|
78
|
+
@user_db = user_db.
|
79
|
+
inject({}) { |h, (user, pwd)|
|
80
|
+
h[user] = AES256.new(pwd); h }
|
73
81
|
end
|
74
82
|
|
75
|
-
def
|
76
|
-
|
83
|
+
def de encrypted_string
|
84
|
+
message = parse_encrypted(encrypted_string)
|
85
|
+
|
86
|
+
if algo = @user_db[message.user]
|
87
|
+
transmission = algo.dec(
|
88
|
+
message.salt,
|
89
|
+
message.iv,
|
90
|
+
message.ciphertext)
|
77
91
|
|
78
|
-
|
79
|
-
|
92
|
+
return transmission
|
93
|
+
else
|
94
|
+
warn "No such user #{message.user} in database; cannot decrypt."
|
95
|
+
return nil
|
80
96
|
end
|
81
97
|
end
|
82
98
|
end
|
99
|
+
|
83
100
|
end
|
84
101
|
|
85
102
|
if $0 == __FILE__
|
86
|
-
enc = EventShipper::
|
103
|
+
enc = EventShipper::Filter::Encryption.new 'password'
|
87
104
|
str = enc.enc 'A very secret text'
|
88
105
|
puts str
|
89
106
|
|
90
|
-
dec = EventShipper::
|
107
|
+
dec = EventShipper::Filter::Encryption.new 'password'
|
91
108
|
puts dec.dec(str)
|
92
109
|
exit
|
93
110
|
|
@@ -95,8 +112,8 @@ if $0 == __FILE__
|
|
95
112
|
puts "Doing it a 1000 times"
|
96
113
|
msg = "foobar"*20
|
97
114
|
puts Benchmark.measure {
|
98
|
-
e = EventShipper::
|
99
|
-
d = EventShipper::
|
115
|
+
e = EventShipper::Filter::Encrypt.new('test', 'password')
|
116
|
+
d = EventShipper::Filter::Decrypt.new('test' => 'password')
|
100
117
|
1000.times do d.call(e.call(msg)) end
|
101
118
|
}
|
102
119
|
|
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
require 'yajl/json_gem'
|
3
|
+
|
4
|
+
require 'event_shipper/protocol'
|
5
|
+
|
6
|
+
module EventShipper::Filter
|
7
|
+
# Serializes a hash by wrapping it into a Protocol::Transmission object
|
8
|
+
# and then serializing it to a string.
|
9
|
+
#
|
10
|
+
class Transmission
|
11
|
+
include EventShipper::Protocol
|
12
|
+
|
13
|
+
def en hash
|
14
|
+
transmission(
|
15
|
+
event(
|
16
|
+
queue: 'queue',
|
17
|
+
json: hash.to_json)).serialize_to_string
|
18
|
+
end
|
19
|
+
def de string
|
20
|
+
transmission = parse_transmission(string)
|
21
|
+
|
22
|
+
if transmission.version != 1
|
23
|
+
warn "Received a transmission with version #{transmission.version}, but this code parses version 1."
|
24
|
+
end
|
25
|
+
|
26
|
+
event = transmission.events.first
|
27
|
+
JSON.parse(event.json)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -18,19 +18,24 @@ module EventShipper
|
|
18
18
|
@thread.abort_on_exception = true
|
19
19
|
end
|
20
20
|
|
21
|
-
def thread_main
|
21
|
+
def thread_main once=false
|
22
22
|
File.open(@path) do |log|
|
23
23
|
log.seek 0, IO::SEEK_END
|
24
24
|
log.extend File::Tail
|
25
25
|
|
26
|
+
log.max_interval = 5
|
27
|
+
log.interval = 1
|
28
|
+
|
26
29
|
log.tail do |line|
|
27
30
|
issue line
|
31
|
+
|
32
|
+
return if once
|
28
33
|
end
|
29
34
|
end
|
30
35
|
end
|
31
36
|
def issue line
|
32
37
|
event = Event.new @host, @path, line
|
33
|
-
@transport.send
|
38
|
+
@transport.send event.to_hash
|
34
39
|
end
|
35
40
|
|
36
41
|
def join
|
@@ -0,0 +1,26 @@
|
|
1
|
+
|
2
|
+
require_relative 'protocol/v1.pb.rb'
|
3
|
+
|
4
|
+
module EventShipper::Protocol
|
5
|
+
def transmission *events
|
6
|
+
Transmission.new(
|
7
|
+
version: 1,
|
8
|
+
events: events)
|
9
|
+
end
|
10
|
+
|
11
|
+
def event attributes={}
|
12
|
+
Event.new(attributes)
|
13
|
+
end
|
14
|
+
def encrypted attributes={}
|
15
|
+
Encrypted.new(attributes)
|
16
|
+
end
|
17
|
+
|
18
|
+
def parse_encrypted string
|
19
|
+
Encrypted.new.tap { |o|
|
20
|
+
o.parse(string) }
|
21
|
+
end
|
22
|
+
def parse_transmission string
|
23
|
+
Transmission.new.tap { |o|
|
24
|
+
o.parse(string) }
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
3
|
+
|
4
|
+
require 'protocol_buffers'
|
5
|
+
|
6
|
+
module EventShipper
|
7
|
+
module Protocol
|
8
|
+
# forward declarations
|
9
|
+
class Encrypted < ::ProtocolBuffers::Message; end
|
10
|
+
class Event < ::ProtocolBuffers::Message; end
|
11
|
+
class Transmission < ::ProtocolBuffers::Message; end
|
12
|
+
|
13
|
+
class Encrypted < ::ProtocolBuffers::Message
|
14
|
+
required :bytes, :salt, 1
|
15
|
+
required :bytes, :iv, 2
|
16
|
+
required :bytes, :ciphertext, 3
|
17
|
+
required :string, :user, 4
|
18
|
+
end
|
19
|
+
|
20
|
+
class Event < ::ProtocolBuffers::Message
|
21
|
+
required :string, :queue, 1
|
22
|
+
required :string, :json, 2
|
23
|
+
end
|
24
|
+
|
25
|
+
class Transmission < ::ProtocolBuffers::Message
|
26
|
+
required :uint32, :version, 1
|
27
|
+
repeated ::EventShipper::Protocol::Event, :events, 2
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
|
2
|
+
// V1 of the on the wire protocol. Use
|
3
|
+
// rprotoc -o lib/event_shipper/protocol/ lib/event_shipper/protocol/v1.proto
|
4
|
+
// to generate this.
|
5
|
+
|
6
|
+
package event_shipper.protocol;
|
7
|
+
|
8
|
+
message Encrypted {
|
9
|
+
required bytes salt = 1;
|
10
|
+
required bytes iv = 2;
|
11
|
+
required bytes ciphertext = 3;
|
12
|
+
required string user = 4;
|
13
|
+
}
|
14
|
+
|
15
|
+
message Event {
|
16
|
+
// The queue to write this event to.
|
17
|
+
required string queue = 1;
|
18
|
+
// A JSON hash of the event
|
19
|
+
required string json = 2;
|
20
|
+
}
|
21
|
+
|
22
|
+
message Transmission {
|
23
|
+
// Should be set to the number 1 to match this file (v1.proto)
|
24
|
+
// Allows for breaking change versioning on top of field versioning.
|
25
|
+
//
|
26
|
+
required uint32 version = 1;
|
27
|
+
|
28
|
+
// Currently, a transmission will only include one event. This may change.
|
29
|
+
repeated Event events = 2;
|
30
|
+
}
|
data/lib/event_shipper/proxy.rb
CHANGED
@@ -3,11 +3,10 @@ require 'redis'
|
|
3
3
|
require 'yaml'
|
4
4
|
require 'clamp'
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
require_relative 'transport/encrypt'
|
9
|
-
require_relative 'transport/udp'
|
6
|
+
require_relative 'filters'
|
7
|
+
require_relative 'udp'
|
10
8
|
|
9
|
+
module EventShipper
|
11
10
|
class Proxy < Clamp::Command
|
12
11
|
|
13
12
|
option %w(-c --config), "FILE", "Location of the configuration file."
|
@@ -20,13 +19,13 @@ module EventShipper
|
|
20
19
|
|
21
20
|
listen = configuration['listen']
|
22
21
|
host, port = listen['host'], listen['port']
|
23
|
-
transport =
|
22
|
+
transport = UDP.new(host, port)
|
24
23
|
|
25
24
|
if configuration['encrypt']
|
26
25
|
users = configuration['users']
|
27
26
|
userdb = Hash[users.map { |str| str.split('/') }]
|
28
27
|
|
29
|
-
transport.
|
28
|
+
transport.wrap Filter::Decrypt.new(userdb)
|
30
29
|
end
|
31
30
|
|
32
31
|
redis_config = configuration['redis']
|
@@ -35,7 +34,7 @@ module EventShipper
|
|
35
34
|
port: redis_config['port'])
|
36
35
|
|
37
36
|
transport.dispatch { |queue, message|
|
38
|
-
# puts message
|
37
|
+
# puts message }
|
39
38
|
redis.lpush queue, message }
|
40
39
|
end
|
41
40
|
end
|
@@ -2,11 +2,10 @@
|
|
2
2
|
require 'yaml'
|
3
3
|
require 'clamp'
|
4
4
|
|
5
|
-
|
5
|
+
require_relative 'udp'
|
6
|
+
require_relative 'log_tailer'
|
6
7
|
|
7
|
-
|
8
|
-
require 'event_shipper/transport/udp'
|
9
|
-
require 'event_shipper/log_tailer'
|
8
|
+
module EventShipper
|
10
9
|
|
11
10
|
class Shipper < Clamp::Command
|
12
11
|
option %w(-c --config), "FILE", "Location of the configuration file."
|
@@ -18,11 +17,11 @@ module EventShipper
|
|
18
17
|
configuration = YAML.load_file(config)
|
19
18
|
|
20
19
|
host, port = configuration['target'].split(':')
|
21
|
-
transport =
|
20
|
+
transport = UDP.new(host, port)
|
22
21
|
|
23
22
|
if configuration['encrypt']
|
24
23
|
user, password = configuration.values_at('user', 'password')
|
25
|
-
transport.
|
24
|
+
transport.wrap Filter::Encrypt.new(user, password)
|
26
25
|
end
|
27
26
|
|
28
27
|
tailers = configuration["logs"].map do |path|
|
@@ -1,17 +1,23 @@
|
|
1
1
|
require 'socket'
|
2
2
|
|
3
|
-
|
3
|
+
require_relative 'filters'
|
4
|
+
|
5
|
+
module EventShipper
|
4
6
|
class UDP
|
5
7
|
def initialize host, port
|
6
8
|
@host, @port = host, port
|
7
9
|
@socket = UDPSocket.new
|
8
|
-
|
9
|
-
|
10
|
+
|
11
|
+
# The Bson-Filter acts as a terminator for hash based messages and
|
12
|
+
# turns things into BSON for the wire.
|
13
|
+
@filters = []
|
14
|
+
wrap Filter::Transmission.new
|
15
|
+
|
10
16
|
@messages_per_period = 0
|
11
17
|
@period_start = Time.now
|
12
18
|
Thread.start do
|
13
19
|
loop do
|
14
|
-
puts "total #{@messages_per_period / (Time.now - @period_start)} msgs/s"
|
20
|
+
puts "total #{(@messages_per_period / (Time.now - @period_start)).round(3)} msgs/s"
|
15
21
|
|
16
22
|
@messages_per_period = 0
|
17
23
|
@period_start = Time.now
|
@@ -21,19 +27,20 @@ module EventShipper::Transport
|
|
21
27
|
end
|
22
28
|
end
|
23
29
|
|
24
|
-
|
30
|
+
def wrap filter
|
31
|
+
@filters << filter
|
32
|
+
end
|
25
33
|
|
26
|
-
def
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
end
|
34
|
+
def encode obj
|
35
|
+
@filters.inject(obj) { |o, f| f.en(o) }
|
36
|
+
end
|
37
|
+
def decode obj
|
38
|
+
@filters.reverse.inject(obj) { |o, f| f.de(o) }
|
32
39
|
end
|
33
40
|
|
34
|
-
def send
|
41
|
+
def send hash
|
35
42
|
@messages_per_period += 1
|
36
|
-
@socket.send
|
43
|
+
@socket.send encode(hash),
|
37
44
|
0, # flags...
|
38
45
|
@host, @port
|
39
46
|
end
|
@@ -46,8 +53,9 @@ module EventShipper::Transport
|
|
46
53
|
datagram, source_info = @socket.recvfrom(10 * 1024)
|
47
54
|
@messages_per_period += 1
|
48
55
|
|
49
|
-
|
50
|
-
|
56
|
+
hash = decode(datagram)
|
57
|
+
|
58
|
+
yield nil, hash
|
51
59
|
end
|
52
60
|
end
|
53
61
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: event-shipper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-05-
|
12
|
+
date: 2013-05-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: redis
|
@@ -91,6 +91,22 @@ dependencies:
|
|
91
91
|
- - ! '>='
|
92
92
|
- !ruby/object:Gem::Version
|
93
93
|
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: ruby-protocol-buffers
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :runtime
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
94
110
|
description: ! "\n event_shipper reads log files and sends each line in logstash
|
95
111
|
JSON format \n via encrypted UDP to redis.\n "
|
96
112
|
email: kaspar.schiess@absurd.li
|
@@ -104,13 +120,20 @@ files:
|
|
104
120
|
- HISTORY
|
105
121
|
- LICENSE
|
106
122
|
- README
|
123
|
+
- lib/event_shipper/benchmark.rb
|
107
124
|
- lib/event_shipper/event.rb
|
125
|
+
- lib/event_shipper/filter/encrypt.rb
|
126
|
+
- lib/event_shipper/filter/transmission.rb
|
127
|
+
- lib/event_shipper/filters.rb
|
108
128
|
- lib/event_shipper/log_tailer.rb
|
129
|
+
- lib/event_shipper/protocol/v1.pb.rb
|
130
|
+
- lib/event_shipper/protocol/v1.proto
|
131
|
+
- lib/event_shipper/protocol.rb
|
109
132
|
- lib/event_shipper/proxy.rb
|
110
133
|
- lib/event_shipper/shipper.rb
|
111
|
-
- lib/event_shipper/
|
112
|
-
- lib/event_shipper/transport/udp.rb
|
134
|
+
- lib/event_shipper/udp.rb
|
113
135
|
- lib/event_shipper.rb
|
136
|
+
- bin/bench
|
114
137
|
- bin/esproxy
|
115
138
|
- bin/esshipper
|
116
139
|
- bin/producer
|