flume-logger 0.1.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in flume-logger.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Lars Sjöström
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# FlumeLogger
+
+This gem implements a subclass of Ruby's Logger class that logs directly to flume. It writes to a flume agent using thrift RPC, support both FlumeNG (default) or FlumeOG (ThriftLegacy).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'flume-logger'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install flume-logger
+
+## Usage
+
+    require 'flume-logger'
+    
+    logger = FlumeLogger.new('localhost', 9090)
+    logger.progname = 'test-flume-ng'
+    logger.info "Hello INFO from ruby"
+    logger.warn "Hello WARN from ruby"
+    logger.error "Hello ERROR from ruby"
+    
+    og_logger = FlumeLogger.new('localhost', 9090, :og)
+    og_logger.progname = 'test-flume-og'
+    og_logger.info "Hello from ruby"
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/flume-logger.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'flume-logger/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "flume-logger"
+  spec.version       = FlumeLogger::VERSION
+  spec.authors       = ["Lars Sjöström"]
+  spec.email         = ["lars@radicore.se"]
+  spec.description   = %q{Ruby logger that writes directly to Flume}
+  spec.summary       = %q{Flume logger for ruby}
+  spec.homepage      = "https://github.com/lsjostro/ruby-flumelogger"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'thrift', '~> 0.9.1'
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+end

data/lib/flume-logger.rb

@@ -0,0 +1,4 @@
+require 'socket'
+require 'flume-logger/version'
+require 'flume-logger/logger'
+require 'flume-logger/eventserver'

data/lib/flume-logger/eventserver.rb

@@ -0,0 +1,42 @@
+class FlumeLogger::Eventserver
+  def initialize(host="localhost", port=9090, flume_type=:ng)
+    @host = host
+    @port = port
+    @client = nil
+    @type = flume_type 
+  end
+  
+  def write(event)
+    begin
+      connect unless @client
+
+      @client.append(event)
+    rescue => e
+      raise
+      warn "#{self.class} - #{e.class} - #{e.message}: #{event}"
+      close
+      @client = nil
+    end
+  end
+
+  def close
+    @client && @transport.close
+  rescue => e
+    warn "#{self.class} - #{e.class} - #{e.message}"
+  end
+  
+  private
+  def connect
+    @client = case @type
+    when :ng then 
+      @transport = Thrift::FramedTransport.new(Thrift::Socket.new(@host, @port))
+      protocol = Thrift::CompactProtocol.new(@transport)
+      ThriftSourceProtocol::Client.new(protocol)
+    when :og then 
+      @transport = Thrift::BufferedTransport.new(Thrift::Socket.new(@host, @port))
+      protocol = Thrift::BinaryProtocol.new(@transport)
+      ThriftFlumeEventServer::Client.new(protocol)
+    end
+    @transport.open()
+  end
+end

data/lib/flume-logger/flume-ng/flume_constants.rb

@@ -0,0 +1,9 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_types'
+

data/lib/flume-logger/flume-ng/flume_types.rb

@@ -0,0 +1,37 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+
+module Status
+  OK = 0
+  FAILED = 1
+  ERROR = 2
+  UNKNOWN = 3
+  VALUE_MAP = {0 => "OK", 1 => "FAILED", 2 => "ERROR", 3 => "UNKNOWN"}
+  VALID_VALUES = Set.new([OK, FAILED, ERROR, UNKNOWN]).freeze
+end
+
+class ThriftFlumeEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  HEADERS = 1
+  BODY = 2
+
+  FIELDS = {
+    HEADERS => {:type => ::Thrift::Types::MAP, :name => 'headers', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}},
+    BODY => {:type => ::Thrift::Types::STRING, :name => 'body', :binary => true}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field headers is unset!') unless @headers
+    raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field body is unset!') unless @body
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+

data/lib/flume-logger/flume-ng/thrift_source_protocol.rb

@@ -0,0 +1,138 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_types'
+
+module ThriftSourceProtocol
+  class Client
+    include ::Thrift::Client
+
+    def append(event)
+      send_append(event)
+      return recv_append()
+    end
+
+    def send_append(event)
+      send_message('append', Append_args, :event => event)
+    end
+
+    def recv_append()
+      result = receive_message(Append_result)
+      return result.success unless result.success.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'append failed: unknown result')
+    end
+
+    def appendBatch(events)
+      send_appendBatch(events)
+      return recv_appendBatch()
+    end
+
+    def send_appendBatch(events)
+      send_message('appendBatch', AppendBatch_args, :events => events)
+    end
+
+    def recv_appendBatch()
+      result = receive_message(AppendBatch_result)
+      return result.success unless result.success.nil?
+      raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'appendBatch failed: unknown result')
+    end
+
+  end
+
+  class Processor
+    include ::Thrift::Processor
+
+    def process_append(seqid, iprot, oprot)
+      args = read_args(iprot, Append_args)
+      result = Append_result.new()
+      result.success = @handler.append(args.event)
+      write_result(result, oprot, 'append', seqid)
+    end
+
+    def process_appendBatch(seqid, iprot, oprot)
+      args = read_args(iprot, AppendBatch_args)
+      result = AppendBatch_result.new()
+      result.success = @handler.appendBatch(args.events)
+      write_result(result, oprot, 'appendBatch', seqid)
+    end
+
+  end
+
+  # HELPER FUNCTIONS AND STRUCTURES
+
+  class Append_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVENT = 1
+
+    FIELDS = {
+      EVENT => {:type => ::Thrift::Types::STRUCT, :name => 'event', :class => ::ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Append_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => ::Status}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+      unless @success.nil? || ::Status::VALID_VALUES.include?(@success)
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!')
+      end
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AppendBatch_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVENTS = 1
+
+    FIELDS = {
+      EVENTS => {:type => ::Thrift::Types::LIST, :name => 'events', :element => {:type => ::Thrift::Types::STRUCT, :class => ::ThriftFlumeEvent}}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class AppendBatch_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    SUCCESS = 0
+
+    FIELDS = {
+      SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success', :enum_class => ::Status}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+      unless @success.nil? || ::Status::VALID_VALUES.include?(@success)
+        raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field success!')
+      end
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+end
+

data/lib/flume-logger/flume-og/flume_compatibility_constants.rb

@@ -0,0 +1,9 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_compatibility_types'
+

data/lib/flume-logger/flume-og/flume_compatibility_types.rb

@@ -0,0 +1,56 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+
+module Priority
+  FATAL = 0
+  ERROR = 1
+  WARN = 2
+  INFO = 3
+  DEBUG = 4
+  TRACE = 5
+  VALUE_MAP = {0 => "FATAL", 1 => "ERROR", 2 => "WARN", 3 => "INFO", 4 => "DEBUG", 5 => "TRACE"}
+  VALID_VALUES = Set.new([FATAL, ERROR, WARN, INFO, DEBUG, TRACE]).freeze
+end
+
+module EventStatus
+  ACK = 0
+  COMMITED = 1
+  ERR = 2
+  VALUE_MAP = {0 => "ACK", 1 => "COMMITED", 2 => "ERR"}
+  VALID_VALUES = Set.new([ACK, COMMITED, ERR]).freeze
+end
+
+class ThriftFlumeEvent
+  include ::Thrift::Struct, ::Thrift::Struct_Union
+  TIMESTAMP = 1
+  PRIORITY = 2
+  BODY = 3
+  NANOS = 4
+  HOST = 5
+  FIELDS = 6
+
+  FIELDS = {
+    TIMESTAMP => {:type => ::Thrift::Types::I64, :name => 'timestamp'},
+    PRIORITY => {:type => ::Thrift::Types::I32, :name => 'priority', :enum_class => ::Priority},
+    BODY => {:type => ::Thrift::Types::STRING, :name => 'body', :binary => true},
+    NANOS => {:type => ::Thrift::Types::I64, :name => 'nanos'},
+    HOST => {:type => ::Thrift::Types::STRING, :name => 'host'},
+    FIELDS => {:type => ::Thrift::Types::MAP, :name => 'fields', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING, :binary => true}}
+  }
+
+  def struct_fields; FIELDS; end
+
+  def validate
+    unless @priority.nil? || ::Priority::VALID_VALUES.include?(@priority)
+      raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field priority!')
+    end
+  end
+
+  ::Thrift::Struct.generate_accessors self
+end
+

data/lib/flume-logger/flume-og/thrift_flume_event_server.rb

@@ -0,0 +1,119 @@
+#
+# Autogenerated by Thrift Compiler (0.9.0)
+#
+# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+#
+
+require 'thrift'
+require 'flume_compatibility_types'
+
+module ThriftFlumeEventServer
+  class Client
+    include ::Thrift::Client
+
+    def append(evt)
+      send_append(evt)
+    end
+
+    def send_append(evt)
+      send_message('append', Append_args, :evt => evt)
+    end
+    def close()
+      send_close()
+      recv_close()
+    end
+
+    def send_close()
+      send_message('close', Close_args)
+    end
+
+    def recv_close()
+      result = receive_message(Close_result)
+      return
+    end
+
+  end
+
+  class Processor
+    include ::Thrift::Processor
+
+    def process_append(seqid, iprot, oprot)
+      args = read_args(iprot, Append_args)
+      @handler.append(args.evt)
+      return
+    end
+
+    def process_close(seqid, iprot, oprot)
+      args = read_args(iprot, Close_args)
+      result = Close_result.new()
+      @handler.close()
+      write_result(result, oprot, 'close', seqid)
+    end
+
+  end
+
+  # HELPER FUNCTIONS AND STRUCTURES
+
+  class Append_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+    EVT = 1
+
+    FIELDS = {
+      EVT => {:type => ::Thrift::Types::STRUCT, :name => 'evt', :class => ::ThriftFlumeEvent}
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Append_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_args
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+  class Close_result
+    include ::Thrift::Struct, ::Thrift::Struct_Union
+
+    FIELDS = {
+
+    }
+
+    def struct_fields; FIELDS; end
+
+    def validate
+    end
+
+    ::Thrift::Struct.generate_accessors self
+  end
+
+end
+

data/lib/flume-logger/logger.rb

@@ -0,0 +1,71 @@
+class FlumeLogger < ::Logger
+  
+  attr_reader :client
+  
+  HOST = ::Socket.gethostname
+  PRIORITY = { "FATAL"   => 0,
+    "CRITICAL" => 0,
+    "ERROR"    => 1,
+    "WARNING"  => 2,
+    "INFO"     => 3,
+    "DEBUG"    => 4,
+    "TRACE"    => 5 }
+  
+  def initialize(host, port, flume_type=:ng, headers={})
+    super(::FlumeLogger::Eventserver.new(host, port, flume_type))
+    @type = flume_type
+    @headers = headers
+  end
+  
+  def add(severity, message = nil, progname = nil, &block)
+    severity ||= UNKNOWN
+    if severity < @level
+      return true
+    end
+    progname ||= @progname
+    if message.nil?
+      if block_given?
+        message = yield
+      else
+        message = progname
+        progname = @progname
+      end
+    end
+    @logdev.write(
+      format_message(format_severity(severity), Time.now, progname, message))
+    true
+  end
+  
+  def format_message(severity, time, progname, message)
+    if progname
+      @headers['application'] = progname
+    end
+
+    event = case @type
+    when :ng
+      $:.push File.expand_path("../flume-ng", __FILE__)
+      require 'thrift_source_protocol'
+
+      @headers['host'] = HOST
+      @headers['pri'] = severity
+      evt = ThriftFlumeEvent.new()
+      evt.headers = @headers
+      evt.body = message
+      evt
+    when :og
+      $:.push File.expand_path("../flume-og", __FILE__)
+      require 'thrift_flume_event_server'  
+
+      evt = ThriftFlumeEvent.new()
+      evt.timestamp = time.to_f.to_i * 1000
+      evt.priority = PRIORITY[severity]
+      evt.body = message
+      evt.host = HOST
+      evt.nanos = time.usec * 1000
+      evt.fields = @headers
+      evt
+    end
+    
+    event
+  end
+end

data/lib/flume-logger/version.rb

@@ -0,0 +1,5 @@
+require 'logger'
+
+class FlumeLogger < ::Logger
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification 
+name: flume-logger
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- "Lars Sj\xC3\xB6str\xC3\xB6m"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-11-11 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thrift
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 9
+        - 1
+        version: 0.9.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 1
+        - 3
+        version: "1.3"
+  type: :development
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+description: Ruby logger that writes directly to Flume
+email: 
+- lars@radicore.se
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- flume-logger.gemspec
+- lib/flume-logger.rb
+- lib/flume-logger/eventserver.rb
+- lib/flume-logger/flume-ng/flume_constants.rb
+- lib/flume-logger/flume-ng/flume_types.rb
+- lib/flume-logger/flume-ng/thrift_source_protocol.rb
+- lib/flume-logger/flume-og/flume_compatibility_constants.rb
+- lib/flume-logger/flume-og/flume_compatibility_types.rb
+- lib/flume-logger/flume-og/thrift_flume_event_server.rb
+- lib/flume-logger/logger.rb
+- lib/flume-logger/version.rb
+has_rdoc: true
+homepage: https://github.com/lsjostro/ruby-flumelogger
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7.1
+signing_key: 
+specification_version: 3
+summary: Flume logger for ruby
+test_files: []
+