loggregator_emitter 2.0.0.pre → 2.0.0

data/README.md

@@ -4,18 +4,18 @@
 
 This gem provides an API to emit messages to the loggregator agent from Ruby applications.
 
-Create an emitter object with the loggregator router host and port, and source type of the emitter.
+Create an emitter object with the loggregator router host and port, a source name of the emitter, and a shared secret (for signing).
 
 Call emit() or emit_error() on this emitter with the application GUID and the message string.
 
-### Valid source types are:
+##### A valid source name is any 3 character string.   Some common component sources are:
 
- 	LogMessage::SourceType::CLOUD_CONTROLLER
- 	LogMessage::SourceType::ROUTER
- 	LogMessage::SourceType::UAA
- 	LogMessage::SourceType::DEA
- 	LogMessage::SourceType::WARDEN_CONTAINER
- 	LogMessage::SourceType::LOGGREGATOR
+ 	API (Cloud Controller)
+ 	RTR (Go Router)
+ 	UAA
+ 	DEA
+ 	APP (Warden container)
+ 	LGR (Loggregator)
 
 ### Setup
 
@@ -27,7 +27,7 @@ Call emit() or emit_error() on this emitter with the application GUID and the me
 
     require "loggregator_emitter"
 
-    emitter = LoggregatorEmitter::Emitter.new("10.10.10.16:38452", LogMessage::SourceType::CLOUD_CONTROLLER)
+    emitter = LoggregatorEmitter::Emitter.new("10.10.10.16:38452", "API")
 
     app_guid = "a8977cb6-3365-4be1-907e-0c878b3a4c6b" # The GUID(UUID) for the user's application
 
@@ -35,6 +35,9 @@ Call emit() or emit_error() on this emitter with the application GUID and the me
 
     emitter.emit_error(app_guid,error_message) # Emits messages with a message type of ERR
 
+### Regenerating Protobuf library
+    protoc --beefcake_out lib/loggregator_messages -I lib/loggregator_messages lib/loggregator_messages/log_message.proto
+
 ### Versioning
 
 This gem is versioned using [semantic versioning](http://semver.org/).

data/lib/loggregator_emitter/emit.rb

@@ -5,14 +5,33 @@ module LoggregatorEmitter
     MAX_MESSAGE_BYTE_SIZE = (9 * 1024) - 512
     TRUNCATED_STRING = "TRUNCATED"
 
-    def initialize(loggregator_server, source_type, source_id = nil, secret=nil)
-      raise ArgumentError, "Must provide valid source type" unless valid_source_type?(source_type)
-
+    SOURCE_TYPE_BY_ID = {
+          1 => "CLOUD_CONTROLLER",
+          2 => "ROUTER",
+          3 => "UAA",
+          4 => "DEA",
+          5 => "WARDEN_CONTAINER",
+          6 => "LOGGREGATOR",
+    }
+
+    def initialize(loggregator_server, source_type_or_name, source_id = nil, secret=nil)
       @host, @port = loggregator_server.split(/:([^:]*$)/)
       raise ArgumentError, "Must provide valid loggregator server: #{loggregator_server}" if !valid_hostname || !valid_port
+      raise ArgumentError, "Must provide valid source_type_or_name: #{source_type_or_name}" unless source_type_or_name
+
+      case source_type_or_name
+        when LogMessage::SourceType::CLOUD_CONTROLLER..LogMessage::SourceType::LOGGREGATOR
+          @source_type = source_type_or_name
+          @source_name = SOURCE_TYPE_BY_ID[source_type_or_name]
+        when String
+          raise ArgumentError, "Custom Source String must be 3 characters" unless source_type_or_name.size == 3
+          @source_type = LogMessage::SourceType::UNKNOWN
+          @source_name = source_type_or_name
+        else
+          raise ArgumentError, "Invalid source: #{source_type_or_name.inspect}  Please use a string or a known source type constant"
+      end
 
       @secret = secret
-      @source_type = source_type
       @source_id = source_id && source_id.to_s
     end
 
@@ -34,10 +53,14 @@ module LoggregatorEmitter
       @host && !@host.match(/:\/\//)
     end
 
+    def split_message(message)
+      message.split(/\n|\r/).reject { |a| a.empty? }
+    end
+
     def emit_message(app_id, message, type)
       return unless app_id && message && message.strip.length > 0
 
-      message.split(/\r\n|\n\r|\n|\r/).each do |m|
+      split_message(message).each do |m|
         if m.bytesize > MAX_MESSAGE_BYTE_SIZE
           m = m.byteslice(0, MAX_MESSAGE_BYTE_SIZE-TRUNCATED_STRING.bytesize) + TRUNCATED_STRING
         end
@@ -46,7 +69,7 @@ module LoggregatorEmitter
           send_protobuffer(create_log_message(app_id, m, type))
         else
           send_protobuffer(create_log_envelope(app_id, m, type))
-	    end
+	      end
       end
     end
 
@@ -57,6 +80,7 @@ module LoggregatorEmitter
       lm.app_id = app_id
       lm.source_id = @source_id
       lm.source_type = @source_type
+      lm.source_name = @source_name
       lm.message_type = type
       lm
     end
@@ -80,14 +104,5 @@ module LoggregatorEmitter
       s.do_not_reverse_lookup = true
       s.sendmsg_nonblock(result, 0, addrinfo_udp)
     end
-
-    def valid_source_type?(source_type)
-      LogMessage::SourceType.constants.each do |name|
-        if LogMessage::SourceType.const_get(name) == source_type
-          return true
-        end
-      end
-      false
-    end
   end
 end

data/lib/loggregator_messages/log_message.pb.rb

@@ -16,6 +16,7 @@ class LogMessage
     DEA = 4
     WARDEN_CONTAINER = 5
     LOGGREGATOR = 6
+    UNKNOWN = 7
   end
 
   required :message, :bytes, 1
@@ -25,6 +26,7 @@ class LogMessage
   required :source_type, LogMessage::SourceType, 5
   optional :source_id, :string, 6
   repeated :drain_urls, :string, 7
+  optional :source_name, :string, 8
 
 end
 

data/lib/loggregator_messages/log_message.proto

@@ -13,6 +13,7 @@ message LogMessage {
         DEA = 4;
         WARDEN_CONTAINER = 5;
         LOGGREGATOR = 6;
+        UNKNOWN = 7;
     }
 
     required bytes message = 1;
@@ -22,6 +23,7 @@ message LogMessage {
     required SourceType source_type = 5;
     optional string source_id = 6;
     repeated string drain_urls = 7;
+    optional string source_name = 8;
 }
 
 message LogEnvelope {

data/lib/loggregator_messages/log_message_extender.rb

@@ -3,14 +3,6 @@ class LogMessage
     {MessageType::OUT => 'STDOUT', MessageType::ERR => 'STDERR'}[message_type]
   end
 
-  def source_type_name
-    {SourceType::CLOUD_CONTROLLER => 'CF[CC]',
-     SourceType::ROUTER => 'CF[Router]',
-     SourceType::UAA => 'CF[UAA]',
-     SourceType::DEA => 'CF[DEA]',
-     SourceType::WARDEN_CONTAINER => 'App',}[source_type]
-  end
-
   def time=(time)
     self.timestamp = (time.tv_sec * 1000000000) + time.tv_nsec
   end

data/lib/symmetric/encryption.rb

@@ -1,3 +1,4 @@
+# coding: US-ASCII
 require 'digest/sha1'
 require 'openssl'
 

data/loggregator_emitter.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "loggregator_emitter"
-  spec.version       = '2.0.0.pre'
+  spec.version       = '2.0.0'
   spec.authors       = ["Pivotal"]
   spec.email         = ["cf-eng@pivotallabs.com"]
   spec.description   = "Library to emit data to Loggregator"

data/spec/loggregator_emitter/emit_performance_spec.rb

@@ -0,0 +1,65 @@
+require "support/fake_loggregator_server"
+require "loggregator_emitter"
+
+class MessageFixture
+  attr_reader :name, :message
+
+  def initialize(name, message, server_threshold, no_server_threshold)
+    @name = name
+    @message = message
+    @server_threshold = server_threshold
+    @no_server_threshold = no_server_threshold
+  end
+
+  def expected(using_server)
+    using_server ? @server_threshold : @no_server_threshold
+  end
+end
+
+class FreePort
+  def self.next_free_port
+    @@next_free_port ||= 12345
+    @@next_free_port += 1
+  end
+end
+
+shared_examples "a performance test" do |fixture, using_server|
+  let(:free_port) { FreePort.next_free_port }
+  let(:iterations) { using_server ? 100 : 1000 }
+  subject(:emitter) { LoggregatorEmitter::Emitter.new("localhost:#{free_port}", "API", 42, "my-secret") }
+
+  before do
+    if using_server
+      @server = FakeLoggregatorServer.new(free_port)
+      @server.start
+    else
+      emitter.should_receive(:send_protobuffer).at_least(iterations).times
+    end
+  end
+
+  after do
+    @server.stop if using_server
+  end
+
+  it "emits #{fixture.name} within a time threshold #{using_server ? 'with' : 'without'} server" do
+    start_time = Time.now.to_f
+
+    iterations.times { emitter.emit("my_app_id", fixture.message) }
+
+    expect(Time.now.to_f - start_time).to be < fixture.expected(using_server)
+  end
+
+end
+
+describe LoggregatorEmitter do
+  @fixtures = []
+  @fixtures << MessageFixture.new("long_message", (124*1024).times.collect { "a" }.join(""), 2.0, 2.0)
+  @fixtures << MessageFixture.new("message with newlines", 10.times.collect { (6*1024).times.collect { "a" }.join("") + "\n" }.join(""), 3.0, 3.0)
+  @fixtures << MessageFixture.new("message worst case", (124*1024).times.collect { "a" }.join("") + "\n", 2.0, 2.0)
+
+  [true, false].each do |using_server|
+    @fixtures.each do |fixture|
+      it_behaves_like "a performance test", fixture, using_server
+    end
+  end
+end

data/spec/loggregator_emitter/emit_spec.rb

@@ -25,6 +25,10 @@ describe LoggregatorEmitter do
       it "is valid if a server name is given" do
         expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", LogMessage::SourceType::DEA) }.not_to raise_error
       end
+
+      it "accepts a string as source type/name" do
+        expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", "STG") }.not_to raise_error
+      end
     end
 
     describe "invalid configurations" do
@@ -32,11 +36,22 @@ describe LoggregatorEmitter do
         it "raises if host has protocol" do
           expect { LoggregatorEmitter::Emitter.new("http://0.0.0.0:12345", LogMessage::SourceType::DEA) }.to raise_error(ArgumentError)
         end
-      end
 
-      describe "error based on source_type" do
-        it "raises if source_type is invalid" do
-          expect { LoggregatorEmitter::Emitter.new("0.0.0.0:12345", 40) }.to raise_error(ArgumentError)
+        it "raises if source is an unknown integer" do
+          expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", 7) }.to raise_error(ArgumentError)
+        end
+
+        it "raises if source is not an integer or string" do
+          expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", nil) }.to raise_error(ArgumentError)
+          expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", 12.0) }.to raise_error(ArgumentError)
+        end
+
+        it "raises if source is too large of a string" do
+          expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", "ABCD") }.to raise_error(ArgumentError)
+        end
+
+        it "raises if source is too small of a string" do
+          expect { LoggregatorEmitter::Emitter.new("some-unknown-address:12345", "AB") }.to raise_error(ArgumentError)
         end
       end
     end
@@ -192,7 +207,7 @@ describe LoggregatorEmitter do
     end
   end
 
-  describe "source id" do
+  describe "source" do
     before do
       @server = FakeLoggregatorServer.new(free_port)
       @server.start
@@ -210,17 +225,28 @@ describe LoggregatorEmitter do
       @server.messages[0]
     end
 
-    it "can be nil" do
+    it "when type is known" do
+      @emitter = LoggregatorEmitter::Emitter.new("0.0.0.0:#{free_port}", LogMessage::SourceType::CLOUD_CONTROLLER)
+      expect(emit_message.source_name).to eq "CLOUD_CONTROLLER"
+    end
+
+    it "when type is unknown" do
+      @emitter = LoggregatorEmitter::Emitter.new("0.0.0.0:#{free_port}", "STG")
+      expect(emit_message.source_name).to eq "STG"
+      expect(emit_message.source_type).to eq LogMessage::SourceType::UNKNOWN
+    end
+
+    it "id can be nil" do
       @emitter = LoggregatorEmitter::Emitter.new("0.0.0.0:#{free_port}", LogMessage::SourceType::CLOUD_CONTROLLER)
       expect(emit_message.source_id).to eq nil
     end
 
-    it "can be passed in as a string" do
+    it "id can be passed in as a string" do
       @emitter = LoggregatorEmitter::Emitter.new("0.0.0.0:#{free_port}", LogMessage::SourceType::CLOUD_CONTROLLER, "some_source_id")
       expect(emit_message.source_id).to eq "some_source_id"
     end
 
-    it "can be passed in as an integer" do
+    it "id can be passed in as an integer" do
       @emitter = LoggregatorEmitter::Emitter.new("0.0.0.0:#{free_port}", LogMessage::SourceType::CLOUD_CONTROLLER, 13)
       expect(emit_message.source_id).to eq "13"
     end

data/spec/loggregator_emitter/log_message_extender_spec.rb

@@ -9,7 +9,7 @@ describe LogMessage do
           :message => "thesearebytes",
           :message_type => 1,
           :timestamp => 3,
-          :source_type => 3
+          :source_type => "UAA"
         })
 
       expect(msg.message_type_name).to eq("STDOUT")
@@ -21,72 +21,13 @@ describe LogMessage do
           :message => "thesearebytes",
           :message_type => 2,
           :timestamp => 3,
-          :source_type => 3
+          :source_type => "UAA"
         })
 
       expect(msg.message_type_name).to eq("STDERR")
     end
   end
 
-  describe "#source_type_name" do
-    it "returns CloudController for type == 1" do
-      msg = LogMessage.new(
-          {
-              :message => "thesearebytes",
-              :message_type => 1,
-              :timestamp => 3,
-              :source_type => 1
-          })
-
-      expect(msg.source_type_name).to eq("CF[CC]")
-    end
-
-    it "returns Router for type == 2" do
-      msg = LogMessage.new(
-          {
-              :message => "thesearebytes",
-              :message_type => 2,
-              :timestamp => 3,
-              :source_type => 2
-          })
-
-      expect(msg.source_type_name).to eq("CF[Router]")
-    end
-    it "returns UAA for type == 3" do
-      msg = LogMessage.new(
-          {
-              :message => "thesearebytes",
-              :message_type => 2,
-              :timestamp => 3,
-              :source_type => 3
-          })
-
-      expect(msg.source_type_name).to eq("CF[UAA]")
-    end
-    it "returns DEA for type == 4" do
-      msg = LogMessage.new(
-          {
-              :message => "thesearebytes",
-              :message_type => 2,
-              :timestamp => 3,
-              :source_type => 4
-          })
-
-      expect(msg.source_type_name).to eq("CF[DEA]")
-    end
-    it "returns WardenContainer for type == 5" do
-      msg = LogMessage.new(
-          {
-              :message => "thesearebytes",
-              :message_type => 2,
-              :timestamp => 3,
-              :source_type => 5
-          })
-
-      expect(msg.source_type_name).to eq("App")
-    end
-  end
-
   describe "#time=" do
     it "takes a time object" do
       msg = LogMessage.new

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: loggregator_emitter
 version: !ruby/object:Gem::Version
-  version: 2.0.0.pre
-  prerelease: 6
+  version: 2.0.0
+  prerelease: 
 platform: ruby
 authors:
 - Pivotal
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-29 00:00:00.000000000 Z
+date: 2013-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: beefcake
@@ -95,6 +95,7 @@ files:
 - lib/loggregator_messages/log_message_extender.rb
 - lib/symmetric/encryption.rb
 - loggregator_emitter.gemspec
+- spec/loggregator_emitter/emit_performance_spec.rb
 - spec/loggregator_emitter/emit_spec.rb
 - spec/loggregator_emitter/encryption_spec.rb
 - spec/loggregator_emitter/log_message_extender_spec.rb
@@ -115,9 +116,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>'
+  - - ! '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
+      segments:
+      - 0
+      hash: 584721548114788876
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.25
@@ -125,6 +129,7 @@ signing_key:
 specification_version: 3
 summary: Library to emit data to Loggregator
 test_files:
+- spec/loggregator_emitter/emit_performance_spec.rb
 - spec/loggregator_emitter/emit_spec.rb
 - spec/loggregator_emitter/encryption_spec.rb
 - spec/loggregator_emitter/log_message_extender_spec.rb