logstash-logger 0.2.1 → 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 289d8d944755e6c3be2857d075b37d8e5c560a45
+  data.tar.gz: 8fdff518b64fdb1c71dc60ba6f9f75183a092831
+SHA512:
+  metadata.gz: dc0992ffd0c6e3478d8018590bcb16fbd436713672c4fc9a6a64595d563002cea11e8f4dae61eb8c99b239f29215213c5ac1dd4dcfccf6ac766a5b241807848c
+  data.tar.gz: ad911e6164b8fc9d53d7f0afb54b68ce0d3f8eef091f4513920cb5fab8645cb5b3366b81c7e7763d491b342140f906bad55a101b341d2857a760252f59789e4c

data/README.md

@@ -6,7 +6,7 @@ writing to a file or syslog since logstash can receive the structured data direc
 
 ## Features
 
-* Writes directly to logstash over a TCP connection.
+* Writes directly to logstash over a UDP or TCP connection.
 * Always writes in logstash JSON format.
 * Logger can take a string message, a hash, a LogStash::Event, or a logstash-formatted json string as input.
 * Events are automatically populated with message, timestamp, host, and severity.
@@ -25,14 +25,23 @@ Or install it yourself as:
 
     $ gem install logstash-logger
 
-## Usage
+## Basic Usage
 
-First set up a logstash agent to receive input over a TCP port.
+First set up a logstash agent to receive input over a UDP or TCP port.
+Then in ruby, create a LogStashLogger that writes to that port.
 
 ```ruby
+require 'logstash-logger'
+
+# Defaults to UDP
 logger = LogStashLogger.new('localhost', 5228)
 logger.info 'test'
-# Logs {"@source":"server-host-name","@tags":[],"@fields":{"severity":"INFO"},"@message":"test","@timestamp":"2012-12-15T00:48:29+00:00"}
+# Writes the following to UDP port 5228:
+# {"@source":"server-host-name","@tags":[],"@fields":{"severity":"INFO"},"@message":"test","@timestamp":"2013-04-08T18:56:23.767273+00:00"}
+
+# Specify UDP or TCP explicitly
+udp_logger = LogStashLogger.new('localhost', 5228, :udp)
+tcp_logger = LogStashLogger.new('localhost', 5229, :tcp)
 ```
 
 ## Rails integration
@@ -49,7 +58,15 @@ To get Rails to nicely output its logs in structured logstash format, try one of
 * [yarder](https://github.com/rurounijones/yarder)
 
 Currently these gems output a JSON string, which LogStashLogger then parses.
-Future versions of these gems could potentially have deeper integration with LogStashLogger.
+Future versions of these gems could potentially have deeper integration with LogStashLogger (i.e. by writing LogStash::Event objects).
+
+## UDP vs TCP
+Should you write to a UDP or TCP listener? It depends on your specific needs, but most applications should use the default (UDP).
+
+* UDP is faster because it's asynchronous (fire-and-forget). However, this means that log messages could get dropped. This is okay for most applications.
+* TCP verifies that every message has been received via two-way communication . This could slow your app down to a crawl if the TCP listener is under heavy load.
+
+For a more detailed discussion of UDP vs TCP, I recommend reading this article: [UDP vs. TCP](http://gafferongames.com/networking-for-game-programmers/udp-vs-tcp/)
 
 ## Ruby compatibility
 
@@ -61,7 +78,18 @@ Verified to work with:
 
 Ruby 1.8.7 is not supported because LogStash::Event is not compatible with Ruby 1.8.7. This might change in the future.
 
-Rubinius might work, but I haven't been able to test it.
+The specs don't pass in Rubinius yet, but the logger does work.
+
+## Breaking change in version 0.3+
+Earlier versions of this gem (<= 0.2.1) only implemented a TCP connection. Newer versions (>= 0.3) also implement UDP, and use that as the new default.
+Please be aware if you are using the default constructor and still require TCP, you should add an additional argument:
+
+```ruby
+# Now defaults to UDP instead of TCP
+logger = LogStashLogger.new('localhost', 5228)
+# Explicitly specify TCP instead of UDP
+logger = LogStashLogger.new('localhost', 5228, :tcp)
+```
 
 ## Contributing
 

data/Rakefile

@@ -2,10 +2,24 @@
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
 
-desc "Run all specs"
+desc "Run all specs with default options"
 RSpec::Core::RakeTask.new(:spec) do |t|
   t.rspec_opts = %w[--color]
   t.verbose = false
 end
 
-task :default => :spec
+desc "Run specs with TCP socket"
+RSpec::Core::RakeTask.new("spec:tcp") do |t|
+  ENV['SOCKET_TYPE'] = 'tcp'
+  t.rspec_opts = %w[--color]
+  t.verbose = false
+end
+
+desc "Run specs with UDP socket"
+RSpec::Core::RakeTask.new("spec:udp") do |t|
+  ENV['SOCKET_TYPE'] = 'udp'
+  t.rspec_opts = %w[--color]
+  t.verbose = false
+end
+
+task :default => ["spec:tcp", "spec:udp"]

data/lib/logstash-logger.rb

@@ -1,10 +1,9 @@
 require 'logstash-logger/version'
 
-require 'thread'
 require 'socket'
 require 'logstash-event'
 
-require 'logstash-logger/tcp_client'
+require 'logstash-logger/socket'
 require 'logstash-logger/logger'
 require 'logstash-logger/version'
 

data/lib/logstash-logger/logger.rb

@@ -3,10 +3,10 @@ class LogStashLogger < ::Logger
   attr_reader :client
   
   LOGSTASH_EVENT_FIELDS = %w(@timestamp @tags @type @source @fields @message).freeze
-  HOST = Socket.gethostname
+  HOST = ::Socket.gethostname
   
-  def initialize(host, port)
-    super(::LogStashLogger::TCPClient.new(host, port))
+  def initialize(host, port, socket_type=:udp)
+    super(::LogStashLogger::Socket.new(host, port, socket_type))
   end
   
   def add(severity, message = nil, progname = nil, &block)

data/lib/logstash-logger/socket.rb

@@ -0,0 +1,34 @@
+class LogStashLogger::Socket
+  def initialize(host, port, socket_type = :udp)
+    @host = host
+    @port = port
+    @type = socket_type
+    @socket = nil
+  end
+  
+  def write(event)
+    begin
+      connect unless @socket
+      
+      @socket.write("#{event.to_hash.to_json}\n")
+    rescue => e
+      warn "#{self.class} - #{e.class} - #{e.message}"
+      close
+      @socket = nil
+    end
+  end
+  
+  def close
+    @socket && @socket.close
+  rescue => e
+    warn "#{self.class} - #{e.class} - #{e.message}"
+  end
+  
+  private
+  def connect
+    @socket = case @type
+    when :udp then UDPSocket.new.tap {|s| s.connect(@host, @port)}
+    when :tcp then TCPSocket.new(@host, @port)
+    end
+  end
+end

data/lib/logstash-logger/version.rb

@@ -1,5 +1,5 @@
 require 'logger'
 
 class LogStashLogger < ::Logger
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/spec/logger_spec.rb

@@ -1,80 +1,130 @@
 require 'spec_helper'
 
 describe LogStashLogger do
-  subject { LogStashLogger.new('localhost', port)}
+  # The type of socket we're testing
+  def socket_type
+    @socket_type ||= (ENV['SOCKET_TYPE'] || 'UDP').to_s.downcase.to_sym
+  end
+  
+  before(:all) { puts "Testing with #{socket_type.to_s.upcase} socket type" }
+  
+  let(:host) { '0.0.0.0' }
+  let(:hostname) { Socket.gethostname }
   let(:port) { 5228 }
-  let(:server) { TCPServer.new(port) }
-  let(:host) { Socket.gethostname }
   
-  before(:all) do
-    server # Initialize TCP server
+  # The logstash logger
+  let(:logger) { LogStashLogger.new(host, port, socket_type) }
+  # The log device that the logger writes to
+  let(:logdev) { logger.instance_variable_get(:@logdev) }
+  
+  let! :listener do
+    case socket_type
+    when :tcp
+      TCPServer.new(port)
+    when :udp
+      UDPSocket.new.tap {|socket| socket.bind(host, port)}
+    end
   end
   
+  # The TCP socket written to by the TCP logstash listener server
+  let(:tcp_client) { listener.accept }
+  
+  # The logstash event to log
   let(:logstash_event) do
-    event = LogStash::Event.new
-    event.message = 'test'
-    event['severity'] = 'INFO'
-    event
+    LogStash::Event.new.tap do |event|
+      event.message = 'test'
+      event['severity'] = 'INFO'
+    end
   end
   
-  def logdev
-    subject.instance_variable_get(:@logdev)
+  # The raw input received by the logstash listener
+  let :listener_input do
+    case socket_type
+    when :tcp then tcp_client.readline
+    when :udp then listener.recvfrom(8192)[0]
+    end
   end
   
-  it 'should use TCPClient as the log device' do
+  # The logstash event received by the listener
+  let(:listener_event) { LogStash::Event.new(JSON.parse listener_input) }
+  
+  #before(:each) do
+    # Sync socket writes so we can receive them in the listener immediately
+    #@socket = logdev.instance_variable_get(:@dev).send(:connect)
+    #@socket.sync = true
+  #end
+  
+  after(:each) do
+    listener.close
+  end
+  
+  # The socket that the logger is writing to
+  #let(:socket) { @socket }
+  
+  it 'uses a LogStashLogger::Socket as the log device' do
     logdev.should be_a Logger::LogDevice
-    logdev.instance_variable_get(:@dev).should be_a LogStashLogger::TCPClient
+    logdev.instance_variable_get(:@dev).should be_a LogStashLogger::Socket
   end
 
-  it 'should take a string message and write a logstash event' do
+  it 'takes a string message as input and writes a logstash event' do
     message = 'test'
-  
-    logdev.should_receive(:write) do |event|
+
+    logdev.should_receive(:write).and_call_original do |event|
       event.should be_a LogStash::Event
-      event.source.should eql(host)
+      event.source.should eql(hostname)
       event.message.should eql(message)
       event['severity'].should eql('INFO')
     end
 
-    subject.info(message)
+    logger.info(message)
+    
+    listener_event.message.should == message
   end
   
-  it 'should take a logstash-formatted json string and write out a logstash event' do
-    logdev.should_receive(:write) do |event|
+  it 'takes a logstash-formatted json string as input and writes out a logstash event' do
+    logdev.should_receive(:write).and_call_original do |event|
       event.should be_a LogStash::Event
       event.message.should eql(logstash_event.message)
-      event.source.should eql(host)
+      event.source.should eql(hostname)
     end
 
-    subject.info(logstash_event.to_json)
+    logger.info(logstash_event.to_json)
+    
+    listener_event.message.should == logstash_event.message
   end
   
-  it 'should take a LogStash::Event and write it out' do
-    logdev.should_receive(:write) do |event|
+  it 'takes a LogStash::Event as input and writes it out intact' do
+    logdev.should_receive(:write).and_call_original do |event|
       event.should be_a LogStash::Event
       event.message.should eql(logstash_event.message)
       event['severity'].should eql(logstash_event['severity'])
       event.timestamp.should eql(logstash_event.timestamp)
-      event.source.should eql(host)
+      event.source.should eql(hostname)
     end
     
-    subject.warn(logstash_event)
+    logger.warn(logstash_event)
+    
+    listener_event.message.should == logstash_event.message
+    listener_event['severity'].should == logstash_event['severity']
   end
   
-  it 'should take a hash and write out a logstash event' do
+  it 'takes a data hash as input and writes out a logstash event' do
     data = {
       "@message" => 'test',
       'severity' => 'INFO'
     }
     
-    logdev.should_receive(:write) do |event|
+    logdev.should_receive(:write).and_call_original do |event|
       event.should be_a LogStash::Event
       event.message.should eql('test')
       event['severity'].should eql('INFO')
-      event.source.should eql(host)
+      event.source.should eql(hostname)
     end
 
-    subject.info(data)
+    logger.info(data.dup)
+    
+    listener_event.message.should == data["@message"]
+    listener_event['severity'].should == data['severity']
   end
   
 end

metadata

@@ -1,62 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: logstash-logger
 version: !ruby/object:Gem::Version
-  version: 0.2.1
-  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors:
 - David Butler
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-12 00:00:00.000000000 Z
+date: 2013-04-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-event
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Ruby logger that writes directly to LogStash
@@ -74,40 +67,33 @@ files:
 - Rakefile
 - lib/logstash-logger.rb
 - lib/logstash-logger/logger.rb
-- lib/logstash-logger/tcp_client.rb
+- lib/logstash-logger/socket.rb
 - lib/logstash-logger/version.rb
 - logstash-logger.gemspec
 - spec/logger_spec.rb
 - spec/spec_helper.rb
 homepage: http://github.com/dwbutler/logstash-logger
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3685342851697372128
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3685342851697372128
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.0
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: LogStash Logger for ruby
 test_files:
 - spec/logger_spec.rb

data/lib/logstash-logger/tcp_client.rb

@@ -1,30 +0,0 @@
-class LogStashLogger::TCPClient
-  def initialize(host, port)
-    @host = host
-    @port = port
-    @socket = nil
-  end
-  
-  def write(event)
-    begin
-      connect unless @socket
-      
-      @socket.write("#{event.to_hash.to_json}\n")
-    rescue => e
-      warn "LogStashLogger::TCPClient - #{e.class} - #{e.message}"
-      close
-      @socket = nil
-    end
-  end
-  
-  def close
-    @socket && @socket.close
-  rescue => e
-    warn "LogStashLogger::TCPClient - #{e.class} - #{e.message}"
-  end
-  
-  private
-  def connect
-    @socket = TCPSocket.new(@host, @port)
-  end
-end