loggregator_emitter 1.0.1 → 2.0.0.pre

data/lib/loggregator_emitter/emit.rb

@@ -5,12 +5,13 @@ module LoggregatorEmitter
     MAX_MESSAGE_BYTE_SIZE = (9 * 1024) - 512
     TRUNCATED_STRING = "TRUNCATED"
 
-    def initialize(loggregator_server, source_type, source_id = nil)
+    def initialize(loggregator_server, source_type, source_id = nil, secret=nil)
       raise ArgumentError, "Must provide valid source type" unless valid_source_type?(source_type)
 
       @host, @port = loggregator_server.split(/:([^:]*$)/)
       raise ArgumentError, "Must provide valid loggregator server: #{loggregator_server}" if !valid_hostname || !valid_port
 
+      @secret = secret
       @source_type = source_type
       @source_id = source_id && source_id.to_s
     end
@@ -36,12 +37,17 @@ module LoggregatorEmitter
     def emit_message(app_id, message, type)
       return unless app_id && message && message.strip.length > 0
 
-      if message.bytesize > MAX_MESSAGE_BYTE_SIZE
-        message = message.byteslice(0, MAX_MESSAGE_BYTE_SIZE-TRUNCATED_STRING.bytesize) + TRUNCATED_STRING
-      end
+      message.split(/\r\n|\n\r|\n|\r/).each do |m|
+        if m.bytesize > MAX_MESSAGE_BYTE_SIZE
+          m = m.byteslice(0, MAX_MESSAGE_BYTE_SIZE-TRUNCATED_STRING.bytesize) + TRUNCATED_STRING
+        end
 
-      lm = create_log_message(app_id, message, type)
-      send_message(lm)
+        if @secret.nil? || @secret.empty?
+          send_protobuffer(create_log_message(app_id, m, type))
+        else
+          send_protobuffer(create_log_envelope(app_id, m, type))
+	    end
+      end
     end
 
     def create_log_message(app_id, message, type)
@@ -55,7 +61,17 @@ module LoggregatorEmitter
       lm
     end
 
-    def send_message(lm)
+    def create_log_envelope(app_id, message, type)
+      crypter = Encryption::Symmetric.new
+      le = LogEnvelope.new()
+      le.routing_key = app_id
+      le.log_message = create_log_message(app_id, message, type)
+      digest = crypter.digest(le.log_message.message)
+      le.signature = crypter.encrypt(@secret, digest)
+      le
+    end
+
+    def send_protobuffer(lm)
       result = lm.encode.buf
       result.unpack("C*")
 

data/lib/loggregator_emitter.rb

@@ -1,3 +1,4 @@
 require "loggregator_emitter/emit"
 require "loggregator_messages/log_message.pb"
 require "loggregator_messages/log_message_extender"
+require "symmetric/encryption"

data/lib/loggregator_messages/log_message.pb.rb

@@ -1,4 +1,4 @@
-## Generated from log_message.proto for logMessage
+## Generated from log_message.proto for logmessage
 require "beefcake"
 
 
@@ -21,7 +21,19 @@ class LogMessage
   required :message, :bytes, 1
   required :message_type, LogMessage::MessageType, 2
   required :timestamp, :sint64, 3
-  optional :app_id, :string, 4
+  required :app_id, :string, 4
   required :source_type, LogMessage::SourceType, 5
   optional :source_id, :string, 6
+  repeated :drain_urls, :string, 7
+
+end
+
+class LogEnvelope
+  include Beefcake::Message
+
+
+  required :routing_key, :string, 1
+  required :signature, :bytes, 2
+  required :log_message, LogMessage, 3
+
 end

data/lib/loggregator_messages/log_message.proto

@@ -1,4 +1,4 @@
-package logMessage;
+package logmessage;
 
 message LogMessage {
     enum MessageType {
@@ -18,7 +18,14 @@ message LogMessage {
     required bytes message = 1;
     required MessageType message_type = 2;
     required sint64 timestamp = 3;
-    optional string app_id = 4;
+    required string app_id = 4;
     required SourceType source_type = 5;
     optional string source_id = 6;
+    repeated string drain_urls = 7;
+}
+
+message LogEnvelope {
+    required string routing_key = 1;
+    required bytes signature = 2;
+    required LogMessage log_message = 3;
 }

data/lib/symmetric/encryption.rb

@@ -0,0 +1,51 @@
+require 'digest/sha1'
+require 'openssl'
+
+module Encryption
+  class Symmetric
+    AES_BLOCKSIZE = 16
+
+    def encrypt(key, message)
+      cipher = OpenSSL::Cipher::AES128.new(:CBC)
+      cipher.encrypt
+      cipher.key = get_encryption_key(key)
+      cipher.padding = 0
+      iv = cipher.random_iv
+
+      iv + cipher.update(pad_buffer(message)) + cipher.final
+    end
+
+    def decrypt(key, encrypted)
+      cipher = OpenSSL::Cipher::AES128.new(:CBC)
+      cipher.padding = 0
+      cipher.decrypt
+      cipher.key = get_encryption_key(key)
+      cipher.iv = encrypted[0..AES_BLOCKSIZE-1]
+
+      unpad_buffer(cipher.update(encrypted[AES_BLOCKSIZE..encrypted.length]) + cipher.final)
+    end
+
+    def digest(value)
+      Digest::SHA256.digest(value)
+    end
+
+    private
+
+    def get_encryption_key(key)
+      digest(key)[0..AES_BLOCKSIZE-1]
+    end
+
+    def pad_buffer(message)
+      bytes_to_pad = AES_BLOCKSIZE - message.length % AES_BLOCKSIZE
+
+      message + "\x80" + "\x00" * (bytes_to_pad - 1)
+    end
+
+    def unpad_buffer(message)
+      raise OpenSSL::Cipher::CipherError unless message.match(/\x80\x00*$/)
+
+      message.gsub(/\x80\x00*$/, '')
+    end
+  end
+end
+

data/loggregator_emitter.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "loggregator_emitter"
-  spec.version       = '1.0.1'
+  spec.version       = '2.0.0.pre'
   spec.authors       = ["Pivotal"]
   spec.email         = ["cf-eng@pivotallabs.com"]
   spec.description   = "Library to emit data to Loggregator"

data/spec/loggregator_emitter/emit_spec.rb

@@ -42,6 +42,60 @@ describe LoggregatorEmitter do
     end
   end
 
+
+  describe "emit_log_envelope" do
+    def make_emitter(host)
+      LoggregatorEmitter::Emitter.new("#{host}:#{free_port}", LogMessage::SourceType::CLOUD_CONTROLLER, 42, "secret")
+    end
+
+    before do
+      @server = FakeLoggregatorServer.new(free_port)
+      @server.start
+    end
+
+    after do
+      @server.stop
+    end
+
+    it "successfully writes envelope protobuffers" do
+      emitter = make_emitter("0.0.0.0")
+      emitter.emit("my_app_id", "Hello there!")
+
+      @server.wait_for_messages(1)
+
+      messages = @server.messages
+
+      expect(messages.length).to eq 1
+      message = messages[0]
+      expect(message.routing_key).to eq "my_app_id"
+
+      actual_digest = Encryption::Symmetric.new.decrypt("secret", message.signature)
+      expected_digest = Encryption::Symmetric.new.digest(message.log_message.message)
+      expect(actual_digest).to eq expected_digest
+
+      expect(message.log_message.message).to eq "Hello there!"
+      expect(message.log_message.app_id).to eq "my_app_id"
+      expect(message.log_message.source_type).to eq LogMessage::SourceType::CLOUD_CONTROLLER
+      expect(message.log_message.source_id).to eq "42"
+      expect(message.log_message.message_type).to eq LogMessage::MessageType::OUT
+    end
+
+    it "makes the right protobuffer" do
+      emitter = make_emitter("0.0.0.0")
+
+      message = nil
+      emitter.stub(:send_protobuffer) do |arg|
+        result = arg.encode.buf
+        message = result.unpack("C*")
+      end
+      emitter.emit("my_app_id", "Hello there!")
+
+      #This test is here to create arrays of bytes to be used in the golang emitter to verify that they are compatible.
+      #One of the results we saw:
+      #[10, 9, 109, 121, 95, 97, 112, 112, 95, 105, 100, 18, 96, 163, 227, 248, 110, 81, 17, 141, 224, 211, 132, 74, 230, 43, 169, 76, 169, 244, 119, 169, 212, 160, 121, 128, 89, 13, 149, 218, 136, 72, 217, 89, 226, 41, 57, 80, 77, 24, 152, 98, 120, 145, 125, 29, 239, 34, 26, 20, 162, 137, 215, 170, 121, 185, 167, 221, 161, 139, 87, 139, 102, 152, 137, 11, 232, 137, 227, 74, 252, 166, 44, 176, 208, 6, 131, 15, 250, 43, 193, 233, 254, 189, 26, 194, 237, 43, 35, 97, 123, 156, 215, 47, 201, 228, 136, 210, 245, 26, 43, 10, 12, 72, 101, 108, 108, 111, 32, 116, 104, 101, 114, 101, 33, 16, 1, 24, 224, 175, 235, 159, 154, 239, 210, 177, 38, 34, 9, 109, 121, 95, 97, 112, 112, 95, 105, 100, 40, 1, 50, 2, 52, 50]
+    end
+  end
+
   {"emit" => LogMessage::MessageType::OUT, "emit_error" => LogMessage::MessageType::ERR}.each do |emit_method, message_type|
     describe "##{emit_method}" do
       def make_emitter(host)
@@ -117,14 +171,24 @@ describe LoggregatorEmitter do
         emitter = make_emitter("localhost")
         message = (124*1024).times.collect { "a" }.join("")
         emitter.send(emit_method, "my_app_id", message)
-        messages = @server.messages
 
         sleep 0.5
 
+        messages = @server.messages
         expect(messages.length).to eq 1
         expect(messages[0].message.bytesize <= LoggregatorEmitter::Emitter::MAX_MESSAGE_BYTE_SIZE).to be_true
         expect(messages[0].message.slice(-9..-1)).to eq("TRUNCATED")
       end
+
+      it "splits messages by newlines" do
+        emitter = make_emitter("localhost")
+        message = "hi\n\rworld\nhow are you\r\ndoing\r"
+        emitter.send(emit_method, "my_app_id", message)
+
+        sleep 0.5
+        messages = @server.messages
+        expect(messages.length).to eq 4
+      end
     end
   end
 
@@ -139,7 +203,7 @@ describe LoggregatorEmitter do
     end
 
     let(:emit_message) do
-       @emitter.emit_error("my_app_id", "Hello there!")
+      @emitter.emit_error("my_app_id", "Hello there!")
 
       @server.wait_for_messages(2)
 

data/spec/loggregator_emitter/encryption_spec.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+
+require "loggregator_emitter"
+
+describe Encryption do
+  subject(:crypter) { Encryption::Symmetric.new }
+  it "pads" do
+    key = "aaaaaaaaaaaaaaaa"
+    message = "1234567890123456"
+    encrypted = crypter.encrypt(key, message)
+    val = encrypted.unpack("C*")
+  end
+
+  it "encrypts" do
+    key = "aaaaaaaaaaaaaaaa"
+    message = "Super secret message that no one should read"
+    encrypted = crypter.encrypt(key, message)
+    decrypted = crypter.decrypt(key, encrypted)
+
+    expect(message).to eq decrypted
+    expect(message).not_to eq encrypted
+  end
+
+  it "encrypts non-deterministicly" do
+    key = "aaaaaaaaaaaaaaaa"
+    message = "Super secret message that no one should read"
+    encrypted1 = crypter.encrypt(key, message)
+    encrypted2 = crypter.encrypt(key, message)
+
+    expect(encrypted1).not_to eq encrypted2
+  end
+
+  it "encrypts with a short key" do
+    key = "short key"
+    message = "Super secret message that no one should read"
+    encrypted = crypter.encrypt(key, message)
+    decrypted = crypter.decrypt(key, encrypted)
+
+    expect(message).to eq decrypted
+    expect(message).not_to eq encrypted
+  end
+
+  it "does not decryption with wrong key" do
+    key = "aaaaaaaaaaaaaaaa"
+    message = "Super secret message that no one should read"
+    encrypted = crypter.encrypt(key, message)
+
+    expect {
+      crypter.decrypt(key + "something went wrong", encrypted)
+    }.to raise_exception(OpenSSL::Cipher::CipherError)
+  end
+
+  describe "compatibility with the encryption done in the Go library github.com/cloudfoundry/loggregatorlib/symmetric" do
+    it "get_encryption_key generates the same key as the go version" do
+      key = "12345"
+      new_key = crypter.send(:get_encryption_key, key)
+
+      expected_hex = [0x59, 0x94, 0x47, 0x1a, 0xbb, 0x1, 0x11, 0x2a, 0xfc, 0xc1, 0x81, 0x59, 0xf6, 0xcc, 0x74, 0xb4]
+
+      expect(new_key.unpack("C*")).to eq expected_hex
+    end
+
+    it "computes digests the same way as the go version" do
+      value = "some-key"
+
+      expected_hex = [0x68, 0x2f, 0x66, 0x97, 0xfa, 0x93, 0xec, 0xa6, 0xc8, 0x1, 0xa2, 0x32, 0x51, 0x9a, 0x9, 0xe3, 0xfe, 0xc, 0x5c, 0x33, 0x94, 0x65, 0xee, 0x53, 0xc3, 0xf9, 0xed, 0xf9, 0x2f, 0xd0, 0x1f, 0x35]
+
+      expect(crypter.digest(value).unpack("C*")).to eq expected_hex
+    end
+  end
+end

data/spec/support/fake_loggregator_server.rb

@@ -29,7 +29,7 @@ class FakeLoggregatorServer
   end
 
   def stop
-    @sockets.each { |socket| socket.close}
+    @sockets.each { |socket| socket.close }
     @threads.each { |thread| Thread.kill(thread) }
   end
 
@@ -42,10 +42,16 @@ class FakeLoggregatorServer
       while true
         begin
           stuff = socket.recv(65536)
-          messages << LogMessage.decode(stuff)
+          decoded_data = LogMessage.decode(stuff.dup)
+          messages << decoded_data
         rescue Beefcake::Message::WrongTypeError, Beefcake::Message::RequiredFieldNotSetError, Beefcake::Message::InvalidValueError => e
-          puts "ERROR"
-          puts e
+          begin
+            decoded_data = LogEnvelope.decode(stuff.dup)
+            messages << decoded_data
+          rescue Beefcake::Message::WrongTypeError, Beefcake::Message::RequiredFieldNotSetError, Beefcake::Message::InvalidValueError => e
+            puts "ERROR: neither envelope nor message extraction worked"
+            puts e
+          end
         end
       end
     end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: loggregator_emitter
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 2.0.0.pre
+  prerelease: 6
 platform: ruby
 authors:
 - Pivotal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-04 00:00:00.000000000 Z
+date: 2013-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beefcake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,6 +30,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -34,6 +38,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -41,6 +46,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -84,35 +93,39 @@ files:
 - lib/loggregator_messages/log_message.pb.rb
 - lib/loggregator_messages/log_message.proto
 - lib/loggregator_messages/log_message_extender.rb
+- lib/symmetric/encryption.rb
 - loggregator_emitter.gemspec
 - spec/loggregator_emitter/emit_spec.rb
+- spec/loggregator_emitter/encryption_spec.rb
 - spec/loggregator_emitter/log_message_extender_spec.rb
 - spec/support/fake_loggregator_server.rb
 homepage: https://www.github.com/cloudfoundry/loggregator_emitter
 licenses:
 - Apache 2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.5
+rubygems_version: 1.8.25
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Library to emit data to Loggregator
 test_files:
 - spec/loggregator_emitter/emit_spec.rb
+- spec/loggregator_emitter/encryption_spec.rb
 - spec/loggregator_emitter/log_message_extender_spec.rb
 - spec/support/fake_loggregator_server.rb

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MzEyMzMwNDA5NTlkZWE5YTM0ZmJkOTQwNTMzYmQyYjljMmM1YzM2NQ==
-  data.tar.gz: !binary |-
-    YmVkNmIwOGM3MDVmZWFhN2Q4OThkYzFmMTRlZjQ1ZTgzY2ViOWQzZQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    M2RlMmQyMDVhYjY0NTJjYWI4OGU3ZGNiMTZmNGZhZWE2ZjI5ZmU1ODRjN2Nj
-    MzQ4ZDVhOWRjYTQwMGE1ODBmZjg0YmQzYmYwYjQ3NDY1ZGMxODBhNDg1MDcx
-    MThjMmQ1ZDE1MGNmMjc3Mjg0MzhjYTljN2EzYWUzOWMyOWU4ZmQ=
-  data.tar.gz: !binary |-
-    OTkxNjgwNjk5ZWYyNTg5M2IxMTI0MmNiZGRjZWZhYzBkNmRhYjUxMmExOTAx
-    YjZmNzU5ODQ2NDE4YmFmODNhYzQ3YjJkY2NjMjJlNGY5M2VjZGMwZGFhZTgw
-    NGQxYTk5ZTRlZDNhMjIxOWIwYTlhZTM1MTliZjNlNzUxNzAyMGY=