lpxc 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bbb9515fbe1b5b1aaae292b1b01b064b716449d9
+  data.tar.gz: a859b4147d2f032283662c2a6066ba73b4f58eaf
+SHA512:
+  metadata.gz: ae76140042c3a5ae27431d1b3b187cbb4094af5bd7579f0350dbc1c047da4dc039ecd94abb6a6d9a2de3e20f2f8f19849e640a1a8195026c2b7eb06e1fd93d73
+  data.tar.gz: ce06660a291346c8fb42ddcfc0d274b75809a9f9c70b79f66b82d397892854d9aab4491577a024dbcc533f07357b4444b0a8de56b77de81117abe2bdc14be0dd

data/lpxc.rb

@@ -0,0 +1,181 @@
+require 'time'
+require 'net/http'
+require 'uri'
+require 'thread'
+require 'timeout'
+
+class Lpxc
+
+  #After parsing opts and initializing defaults, the initializer
+  #will start 2 threads. One thread for sending HTTP requests and another
+  #thread for flusing log messages to the outlet thread periodically.
+  #:hash => {}:: A data structure for grouping log messages by token.
+  #:request_queue => SizedQueue.new::  Contains HTTP requests ready for outlet thread to deliver to logplex.
+  #:default_token => nil:: You can specify a token that will be used for any call to Lpxc#puts that doesn't include a token.
+  #:structured_data => '-':: Structured-data field for syslog headers. Ignored by logplex.
+  #:msgid => '-'::  Msg ID field for syslog headers. Ignored by logplex.
+  #:procid => 'lpxc':: Proc ID field for syslog headers. This will show up in the Heroku logs tail command as: app [lpxc].
+  #:hostname => 'myhost':: Hostname field for syslog headers. Ignored by logplex.
+  #:max_reqs_per_conn => 1_000:: Number of requests before we re-establish our keep-alive connection to logplex.
+  #:conn_timeout => 2:: Number of seconds before timing out a sindle request to logplex.
+  #:batch_size => 300:: Max number of log messages inside single HTTP request.
+  #:flush_interval => 0.5:: Fractional number of seconds before flushing all log messages in buffer to logplex.
+  #:logplex_url => \'https://east.logplex.io/logs':: HTTP server that will accept our log messages.
+  def initialize(opts={})
+    @hash_lock = Mutex.new
+    @hash = opts[:hash] || Hash.new
+    @request_queue = opts[:request_queue] || SizedQueue.new(1)
+    @default_token = opts[:default_token]
+    @structured_data = opts[:structured_data] || "-"
+    @msgid = opts[:msgid] || "-"
+    @procid = opts[:procid] || "lpxc"
+    @hostname = opts[:hostname] || "myhost"
+    @max_reqs_per_conn = opts[:max_reqs_per_conn] || 1_000
+    @conn_timeout = opts[:conn_timeout] || 2
+    @batch_size = opts[:batch_size] || 300
+    @flush_interval = opts[:flush_interval] || 0.5
+    @logplex_url = URI(opts[:logplex_url] || ENV["LOGPLEX_URL"] || 
+      raise("Must set logplex url."))
+
+    #Keep track of the number of requests that the outlet
+    #is processing. This value is used by the wait function.
+    @req_in_flight = 0
+
+    #Initialize the last_flush to an arbitrary time.
+    @last_flush = Time.now + @flush_interval
+
+    #Start the processing threads.
+    Thread.new {outlet}
+    Thread.new {delay_flush}
+  end
+
+  #The interface to publish logs into the stream.
+  #This function will set the log message to the current time in UTC.
+  #If the buffer for this token's log messages is full, it will flush the buffer.
+  def puts(msg, tok=@default_token)
+    @hash_lock.synchronize do
+      #Messages are grouped by their token since 1 http request
+      #to logplex must only contain log messages belonging to a single token.
+      q = @hash[tok] ||= SizedQueue.new(@batch_size)
+      #This call will block if the queue is full.
+      #However this should never happen since the next command will flush
+      #the queue if we add the last item.
+      q.enq({:t => Time.now.utc, :token => tok, :msg => msg})
+      flush if q.size == q.max
+    end
+  end
+
+  #Wait until all of the data has been cleared from memory.
+  #This is useful if you don't want your program to exit before
+  #we are able to deliver log messages to logplex.
+  def wait
+    sleep(0.1) until
+      @hash.length.zero? &&
+      @request_queue.empty? &&
+      @req_in_flight.zero?
+  end
+
+  private
+
+  #Take a lock to read all of the buffered messages.
+  #Once we have read the messages, we make 1 http request for the batch.
+  #We pass the request off into the request queue so that the request
+  #can be sent to LOGPLEX_URL.
+  def flush
+    @hash.each do |tok, msgs|
+      #Copy the messages from the queue into the payload array.
+      payloads = []
+      msgs.size.times {payloads << msgs.deq}
+      return if payloads.nil? || payloads.empty?
+
+      #Use the payloads array to build a string that will be
+      #used as the http body for the logplex request.
+      body = ""
+      payloads.flatten.each do |payload|
+        body += "#{fmt(payload)}"
+      end
+
+      #Build a new HTTP request and place it into the queue
+      #to be processed by the HTTP connection.
+      req = Net::HTTP::Post.new(@logplex_url.path)
+      req.basic_auth("token", tok)
+      req.add_field('Content-Type', 'application/logplex-1')
+      req.body = body
+      @request_queue.enq(req)
+      @hash.delete(tok)
+      @last_flush = Time.now
+    end
+  end
+
+
+  #This method must be called in order for the messages to be sent to Logplex.
+  #This method also spawns a thread that allows the messages to be batched.
+  #Messages are flushed from memory every 500ms or when we have 300 messages,
+  #whichever comes first.
+  def delay_flush
+    loop do
+      begin
+        if interval_ready?
+          @hash_lock.synchronize {flush}
+        end
+        sleep(0.01)
+      rescue => e
+        $stderr.puts("at=start-error error=#{e.message}") if ENV['DEBUG']
+      end
+    end
+  end
+
+  def interval_ready?
+    (Time.now.to_f - @last_flush.to_f).abs >= @flush_interval
+  end
+
+  #Format the user message into RFC5425 format.
+  #This method also prepends the length to the message.
+  def fmt(data)
+    pkt = "<190>1 "
+    pkt += "#{data[:t].strftime("%Y-%m-%dT%H:%M:%S+00:00")} "
+    pkt += "#{@hostname} "
+    pkt += "#{data[:token]} "
+    pkt += "#{@procid} "
+    pkt += "#{@msgid} "
+    pkt += "#{@structured_data} "
+    pkt += data[:msg]
+    "#{pkt.size} #{pkt}"
+  end
+
+  #We use a keep-alive connection to send data to LOGPLEX_URL.
+  #Each request will contain one or more log messages.
+  def outlet
+    loop do
+      http = Net::HTTP.new(@logplex_url.host, @logplex_url.port)
+      http.set_debug_output($stdout) if ENV['DEBUG']
+      http.use_ssl = true if @logplex_url.scheme == 'https'
+      begin
+        http.start do |conn|
+          num_reqs = 0
+          while num_reqs < @max_reqs_per_conn
+            #Blocks waiting for a request.
+            req = @request_queue.deq
+            @req_in_flight += 1
+            resp = nil
+            begin
+              Timeout::timeout(@conn_timeout) {resp = conn.request(req)}
+            rescue => e
+              $stdout.puts("at=req-error msg=#{e.message}") if ENV['DEBUG']
+              next
+            ensure
+              @req_in_flight -= 1
+            end
+            num_reqs += 1
+            $stdout.puts("at=req-sent status=#{resp.code}") if ENV['DEBUG']
+          end
+        end
+      rescue => e
+        $stdout.puts("at=req-error msg=#{e.message}") if ENV['DEBUG']
+      ensure
+        http.finish
+      end
+    end
+  end
+
+end

data/readme.md

@@ -0,0 +1,17 @@
+#lpxc
+
+A simple ruby client to help send log messages to Heroku's logplex.
+
+## Usage
+
+```ruby
+require 'lpxc'
+lpxc = Lpxc.new
+lpxc.puts("hello world", 't.123')
+```
+
+## Runing Tests
+
+```bash
+$ ruby test.rb
+```

metadata

@@ -0,0 +1,45 @@
+--- !ruby/object:Gem::Specification
+name: lpxc
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ryan Smith (♠ ace hacker)
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-29 00:00:00.000000000 Z
+dependencies: []
+description: Ruby client for sending data to Heroku's logplex.
+email: r@32k.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- readme.md
+- lpxc.rb
+homepage: http://github.com/ryandotsmith/lpxc
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- .
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.2
+signing_key: 
+specification_version: 4
+summary: Ruby client for logplex.
+test_files: []