redis-slowlog-stasher 0.0.3

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2014, 3dna Corp
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the 3dna Corp nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

data/README.md

@@ -0,0 +1,27 @@
+redis-slowlog-stasher
+
+# what?
+
+[redis](http://redis.io) has this awesome feature called the [slowlog](http://redis.io/commands/slowlog), which keeps track of commands that take a long time to run.
+
+# why?
+
+because redis is single threaded, if you have a command that takes too long, it'll block all other commands, and on a high traffic redis server, this can be disastrous.
+
+# how?
+
+funny enough, this program is designed to use [rabbitmq](http://www.rabbitmq.com/)as a transport for the logs. Why rabbitmq? Because I already have a rabbitmq set up for "reliable" log transport and that's what I'm going to keep using for now.
+
+The basic idea, though, is that it calls the `slowlog` command on the redis server periodically and any new messages in the slowlog get sent over to logstash. Pretty simple, right?
+
+# who?
+
+This script was written by Jeremy Kitchen while working at NationBuilder in May, 2014
+
+# license
+
+I consider this to be a pretty trivial thing, so I won't be making any claims of copyright. Please feel free to use, modify, steal, sell, use for bad things, whatever.
+
+In jurisdictions where applicable, I'm placing this code into the public domain. Where it's not I'm granting a non-exclusive non-revokable royalty free license to use the code as you please.
+
+If you like it, drop me a [gittip](https://www.gittip.com/kitchen) or buy me a beverage some time.

data/bin/redis-slowlog-stasher

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'redis_slowlog_stasher'
+
+
+require 'time'
+require 'redis'
+require 'bunny'
+require 'uri'
+require 'json'
+
+
+options = RedisSlowlogStasher::Argparser.parse(ARGV)
+logger = options.logger
+redis_arg, rabbit_arg = ARGV
+
+# methinks this logic should just be in the StateFile?
+# if you pass it a nil filename write is a noop and read gives you 0,0?
+if options.state_file
+  previous_entry_timestamp, previous_entry_id = options.state_file.read
+else
+  previous_entry_timestamp, previous_entry_id = [0,0]
+end
+
+redis_uri = URI.parse(redis_arg)
+redis = Redis.new(host: redis_uri.host, port: redis_uri.port, password: redis_uri.user)
+
+rabbit = Bunny.new(rabbit_arg)
+rabbit.start
+channel = rabbit.create_channel
+exchange = channel.direct(options.exchange)
+
+base_event_object = {}
+if options.type
+  base_event_object.merge!({'type' => options.type})
+end
+if options.tags
+  base_event_object.merge!({'tags' => options.tags})
+end
+if options.fields
+  base_event_object.merge!(options.fields)
+end
+
+# start the main loop
+loop do
+  logger.debug("loop iteration")
+  slowlog_entries = redis.slowlog('get',options.check_entries)
+
+  slowlog_entries.reverse.each do |slowlog_entry|
+    # "parse" the entry
+    entry_id, entry_timestamp, duration, args = slowlog_entry
+    command = args.shift
+
+    #logger.debug("previous_entry_id: #{previous_entry_id}, entry_id: #{entry_id}; previous_entry_timestamp: #{previous_entry_timestamp}, entry_timestamp: #{entry_timestamp}")
+
+    # TODO: combine all of these into one ugly if
+    if (entry_id > previous_entry_id) || (entry_id < previous_entry_id && entry_timestamp >= previous_entry_timestamp) || (entry_id == previous_entry_id && entry_timestamp != previous_entry_timestamp)
+      logger.info("new entry id #{entry_id} duration #{duration} with timestamp #{entry_timestamp}: #{command} #{args}")
+      entry_hash = base_event_object.merge({
+        '@timestamp' => Time.at(entry_timestamp).utc.iso8601,
+        'version' => 1,
+        'duration' => duration,
+        'command' => command,
+        'args' => args,
+      })
+
+      logger.debug(entry_hash.to_json)
+      # log it
+      exchange.publish(entry_hash.to_json, routing_key: options.routing_key)
+
+      # update trackers
+      previous_entry_timestamp = entry_timestamp
+      previous_entry_id = entry_id
+      if options.state_file
+        options.state_file.write(entry_timestamp, entry_id)
+      end
+    end
+  end
+
+  sleep options.check_interval
+end
+

data/lib/redis_slowlog_stasher.rb

@@ -0,0 +1,6 @@
+require 'redis_slowlog_stasher/statefile'
+require 'redis_slowlog_stasher/version'
+require 'redis_slowlog_stasher/argparser'
+
+module RedisSlowlogStasher
+end

data/lib/redis_slowlog_stasher/argparser.rb

@@ -0,0 +1,88 @@
+require 'ostruct'
+require 'optparse'
+module RedisSlowlogStasher
+  class Argparser
+    def self.parse(args)
+      options = OpenStruct.new
+
+      options.check_interval = 10
+      options.check_entries = 25
+      options.exchange = 'logstash'
+      options.routing_key = 'logstash'
+
+      options.logger = Logger.new(STDOUT)
+
+      opts = OptionParser.new do |opts|
+        opts.banner = 'Usage: redis-slowlog-stasher [options] redis-server rabbitmq-amqp-uri'
+
+        opts.separator ""
+        opts.separator "options:"
+
+        opts.on("--exchange EXCHANGE", String, "the name of the exchange to use") do |exchange|
+          options.exchange = exchange
+        end
+
+        opts.on("--routing-key KEY", String, "the routing key to use") do |routing_key|
+          options.routing_key = routing_key
+        end
+
+        opts.on("--state-file FILE", String, "the path to the state file which tells this program which",
+                "slowlog entry was the last one processed so we can attempt",
+                "to not lose any messages and not duplicate any, either.") do |state_file|
+          options.state_file = StateFile.new(state_file)
+        end
+
+        opts.on("--type TYPE", String, "the value of the type field of the event") do | type |
+          options.type = type
+        end
+
+        opts.on("--tags tag1,tag2,tag3", Array, "a list of tags to add to the event") do | tags |
+          options.tags ||= []
+          options.tags.concat(tags)
+        end
+
+        opts.on("--add-field field1=value1,field2=value2", Array, "a list of option fields and their values to add to the event") do |fields|
+          options.fields ||= {}
+          options.fields.merge!(Hash[fields.map { |field| field.split('=', 2) }])
+        end
+
+        opts.on("--check-interval SECONDS", Float, "how frequently to check the slowlog for new entries. In seconds") do |check_interval|
+          options.check_interval = check_interval
+        end
+
+        opts.on("--check-entries COUNT", Integer, "how many entries to check for each interval.") do |check_entries|
+          options.check_entries = check_entries
+        end
+
+        opts.on("--log-level LEVEL", ["FATAL","ERROR","WARN","INFO","DEBUG"], "sets the level of output verbosity") do |log_level|
+          case log_level
+          when "FATAL"
+            logger.level = Logger::FATAL
+          when "ERROR"
+            logger.level = Logger::ERROR
+          when "WARN"
+            logger.level = Logger::WARN
+          when "INFO"
+            logger.level = Logger::INFO
+          when "DEBUG"
+            logger.level = Logger::DEBUG
+          end
+        end
+
+        opts.on_tail("-h", "--help", "Show this message") do
+          puts opts
+          exit
+        end
+
+        opts.on_tail("--version", "Show version") do
+          puts '0.0.1'
+          exit
+        end
+
+      end
+
+      opts.parse!(args)
+      options
+    end
+  end
+end

data/lib/redis_slowlog_stasher/statefile.rb

@@ -0,0 +1,21 @@
+# TODO: if you give it a nil filename, read gives you 0,0, write is a noop?
+class StateFile
+  def initialize(filename)
+    @filename = filename
+  end
+
+  def read
+    File.open(@filename, 'r') do |state_file|
+      state_file.gets.chomp.split(':').map(&:to_i)
+    end
+  rescue Errno::ENOENT
+    # if the file doesn't exist, just return the thing
+    [0,0]
+  end
+
+  def write(entry_timestamp, entry_id)
+    File.open(@filename, 'w') do |state_file|
+      state_file.puts("#{entry_timestamp}:#{entry_id}")
+    end
+  end
+end

data/lib/redis_slowlog_stasher/version.rb

@@ -0,0 +1,3 @@
+module RedisSlowlogStasher
+  VERSION = '0.0.3'
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification
+name: redis-slowlog-stasher
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+  prerelease: 
+platform: ruby
+authors:
+- Jeremy Kitchen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: watches the redis slowlog command and sends structured slowlog entries
+  to logstash for processing and storage
+email: jeremy@nationbuilder.com
+executables:
+- redis-slowlog-stasher
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/redis_slowlog_stasher.rb
+- lib/redis_slowlog_stasher/statefile.rb
+- lib/redis_slowlog_stasher/argparser.rb
+- lib/redis_slowlog_stasher/version.rb
+- bin/redis-slowlog-stasher
+- README.md
+- LICENSE
+homepage: https://github.com/3dna/redis-slowlog-stasher
+licenses:
+- BSD-3-Clause
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: put redis slowlog entries into logstash
+test_files: []
+has_rdoc: