redis-slowlog-stasher 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/LICENSE +27 -0
- data/README.md +27 -0
- data/bin/redis-slowlog-stasher +82 -0
- data/lib/redis_slowlog_stasher.rb +6 -0
- data/lib/redis_slowlog_stasher/argparser.rb +88 -0
- data/lib/redis_slowlog_stasher/statefile.rb +21 -0
- data/lib/redis_slowlog_stasher/version.rb +3 -0
- metadata +87 -0
data/LICENSE
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
Copyright (c) 2014, 3dna Corp
|
2
|
+
All rights reserved.
|
3
|
+
|
4
|
+
Redistribution and use in source and binary forms, with or without
|
5
|
+
modification, are permitted provided that the following conditions are met:
|
6
|
+
|
7
|
+
* Redistributions of source code must retain the above copyright notice, this
|
8
|
+
list of conditions and the following disclaimer.
|
9
|
+
|
10
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
11
|
+
this list of conditions and the following disclaimer in the documentation
|
12
|
+
and/or other materials provided with the distribution.
|
13
|
+
|
14
|
+
* Neither the name of the 3dna Corp nor the names of its
|
15
|
+
contributors may be used to endorse or promote products derived from
|
16
|
+
this software without specific prior written permission.
|
17
|
+
|
18
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
19
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
20
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
21
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
22
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
23
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
24
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
25
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
26
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
27
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
redis-slowlog-stasher
|
2
|
+
|
3
|
+
# what?
|
4
|
+
|
5
|
+
[redis](http://redis.io) has this awesome feature called the [slowlog](http://redis.io/commands/slowlog), which keeps track of commands that take a long time to run.
|
6
|
+
|
7
|
+
# why?
|
8
|
+
|
9
|
+
because redis is single threaded, if you have a command that takes too long, it'll block all other commands, and on a high traffic redis server, this can be disastrous.
|
10
|
+
|
11
|
+
# how?
|
12
|
+
|
13
|
+
funny enough, this program is designed to use [rabbitmq](http://www.rabbitmq.com/)as a transport for the logs. Why rabbitmq? Because I already have a rabbitmq set up for "reliable" log transport and that's what I'm going to keep using for now.
|
14
|
+
|
15
|
+
The basic idea, though, is that it calls the `slowlog` command on the redis server periodically and any new messages in the slowlog get sent over to logstash. Pretty simple, right?
|
16
|
+
|
17
|
+
# who?
|
18
|
+
|
19
|
+
This script was written by Jeremy Kitchen while working at NationBuilder in May, 2014
|
20
|
+
|
21
|
+
# license
|
22
|
+
|
23
|
+
I consider this to be a pretty trivial thing, so I won't be making any claims of copyright. Please feel free to use, modify, steal, sell, use for bad things, whatever.
|
24
|
+
|
25
|
+
In jurisdictions where applicable, I'm placing this code into the public domain. Where it's not I'm granting a non-exclusive non-revokable royalty free license to use the code as you please.
|
26
|
+
|
27
|
+
If you like it, drop me a [gittip](https://www.gittip.com/kitchen) or buy me a beverage some time.
|
@@ -0,0 +1,82 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'redis_slowlog_stasher'
|
4
|
+
|
5
|
+
|
6
|
+
require 'time'
|
7
|
+
require 'redis'
|
8
|
+
require 'bunny'
|
9
|
+
require 'uri'
|
10
|
+
require 'json'
|
11
|
+
|
12
|
+
|
13
|
+
options = RedisSlowlogStasher::Argparser.parse(ARGV)
|
14
|
+
logger = options.logger
|
15
|
+
redis_arg, rabbit_arg = ARGV
|
16
|
+
|
17
|
+
# methinks this logic should just be in the StateFile?
|
18
|
+
# if you pass it a nil filename write is a noop and read gives you 0,0?
|
19
|
+
if options.state_file
|
20
|
+
previous_entry_timestamp, previous_entry_id = options.state_file.read
|
21
|
+
else
|
22
|
+
previous_entry_timestamp, previous_entry_id = [0,0]
|
23
|
+
end
|
24
|
+
|
25
|
+
redis_uri = URI.parse(redis_arg)
|
26
|
+
redis = Redis.new(host: redis_uri.host, port: redis_uri.port, password: redis_uri.user)
|
27
|
+
|
28
|
+
rabbit = Bunny.new(rabbit_arg)
|
29
|
+
rabbit.start
|
30
|
+
channel = rabbit.create_channel
|
31
|
+
exchange = channel.direct(options.exchange)
|
32
|
+
|
33
|
+
base_event_object = {}
|
34
|
+
if options.type
|
35
|
+
base_event_object.merge!({'type' => options.type})
|
36
|
+
end
|
37
|
+
if options.tags
|
38
|
+
base_event_object.merge!({'tags' => options.tags})
|
39
|
+
end
|
40
|
+
if options.fields
|
41
|
+
base_event_object.merge!(options.fields)
|
42
|
+
end
|
43
|
+
|
44
|
+
# start the main loop
|
45
|
+
loop do
|
46
|
+
logger.debug("loop iteration")
|
47
|
+
slowlog_entries = redis.slowlog('get',options.check_entries)
|
48
|
+
|
49
|
+
slowlog_entries.reverse.each do |slowlog_entry|
|
50
|
+
# "parse" the entry
|
51
|
+
entry_id, entry_timestamp, duration, args = slowlog_entry
|
52
|
+
command = args.shift
|
53
|
+
|
54
|
+
#logger.debug("previous_entry_id: #{previous_entry_id}, entry_id: #{entry_id}; previous_entry_timestamp: #{previous_entry_timestamp}, entry_timestamp: #{entry_timestamp}")
|
55
|
+
|
56
|
+
# TODO: combine all of these into one ugly if
|
57
|
+
if (entry_id > previous_entry_id) || (entry_id < previous_entry_id && entry_timestamp >= previous_entry_timestamp) || (entry_id == previous_entry_id && entry_timestamp != previous_entry_timestamp)
|
58
|
+
logger.info("new entry id #{entry_id} duration #{duration} with timestamp #{entry_timestamp}: #{command} #{args}")
|
59
|
+
entry_hash = base_event_object.merge({
|
60
|
+
'@timestamp' => Time.at(entry_timestamp).utc.iso8601,
|
61
|
+
'version' => 1,
|
62
|
+
'duration' => duration,
|
63
|
+
'command' => command,
|
64
|
+
'args' => args,
|
65
|
+
})
|
66
|
+
|
67
|
+
logger.debug(entry_hash.to_json)
|
68
|
+
# log it
|
69
|
+
exchange.publish(entry_hash.to_json, routing_key: options.routing_key)
|
70
|
+
|
71
|
+
# update trackers
|
72
|
+
previous_entry_timestamp = entry_timestamp
|
73
|
+
previous_entry_id = entry_id
|
74
|
+
if options.state_file
|
75
|
+
options.state_file.write(entry_timestamp, entry_id)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
sleep options.check_interval
|
81
|
+
end
|
82
|
+
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'ostruct'
|
2
|
+
require 'optparse'
|
3
|
+
module RedisSlowlogStasher
|
4
|
+
class Argparser
|
5
|
+
def self.parse(args)
|
6
|
+
options = OpenStruct.new
|
7
|
+
|
8
|
+
options.check_interval = 10
|
9
|
+
options.check_entries = 25
|
10
|
+
options.exchange = 'logstash'
|
11
|
+
options.routing_key = 'logstash'
|
12
|
+
|
13
|
+
options.logger = Logger.new(STDOUT)
|
14
|
+
|
15
|
+
opts = OptionParser.new do |opts|
|
16
|
+
opts.banner = 'Usage: redis-slowlog-stasher [options] redis-server rabbitmq-amqp-uri'
|
17
|
+
|
18
|
+
opts.separator ""
|
19
|
+
opts.separator "options:"
|
20
|
+
|
21
|
+
opts.on("--exchange EXCHANGE", String, "the name of the exchange to use") do |exchange|
|
22
|
+
options.exchange = exchange
|
23
|
+
end
|
24
|
+
|
25
|
+
opts.on("--routing-key KEY", String, "the routing key to use") do |routing_key|
|
26
|
+
options.routing_key = routing_key
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on("--state-file FILE", String, "the path to the state file which tells this program which",
|
30
|
+
"slowlog entry was the last one processed so we can attempt",
|
31
|
+
"to not lose any messages and not duplicate any, either.") do |state_file|
|
32
|
+
options.state_file = StateFile.new(state_file)
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("--type TYPE", String, "the value of the type field of the event") do | type |
|
36
|
+
options.type = type
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on("--tags tag1,tag2,tag3", Array, "a list of tags to add to the event") do | tags |
|
40
|
+
options.tags ||= []
|
41
|
+
options.tags.concat(tags)
|
42
|
+
end
|
43
|
+
|
44
|
+
opts.on("--add-field field1=value1,field2=value2", Array, "a list of option fields and their values to add to the event") do |fields|
|
45
|
+
options.fields ||= {}
|
46
|
+
options.fields.merge!(Hash[fields.map { |field| field.split('=', 2) }])
|
47
|
+
end
|
48
|
+
|
49
|
+
opts.on("--check-interval SECONDS", Float, "how frequently to check the slowlog for new entries. In seconds") do |check_interval|
|
50
|
+
options.check_interval = check_interval
|
51
|
+
end
|
52
|
+
|
53
|
+
opts.on("--check-entries COUNT", Integer, "how many entries to check for each interval.") do |check_entries|
|
54
|
+
options.check_entries = check_entries
|
55
|
+
end
|
56
|
+
|
57
|
+
opts.on("--log-level LEVEL", ["FATAL","ERROR","WARN","INFO","DEBUG"], "sets the level of output verbosity") do |log_level|
|
58
|
+
case log_level
|
59
|
+
when "FATAL"
|
60
|
+
logger.level = Logger::FATAL
|
61
|
+
when "ERROR"
|
62
|
+
logger.level = Logger::ERROR
|
63
|
+
when "WARN"
|
64
|
+
logger.level = Logger::WARN
|
65
|
+
when "INFO"
|
66
|
+
logger.level = Logger::INFO
|
67
|
+
when "DEBUG"
|
68
|
+
logger.level = Logger::DEBUG
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
73
|
+
puts opts
|
74
|
+
exit
|
75
|
+
end
|
76
|
+
|
77
|
+
opts.on_tail("--version", "Show version") do
|
78
|
+
puts '0.0.1'
|
79
|
+
exit
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
opts.parse!(args)
|
85
|
+
options
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# TODO: if you give it a nil filename, read gives you 0,0, write is a noop?
|
2
|
+
class StateFile
|
3
|
+
def initialize(filename)
|
4
|
+
@filename = filename
|
5
|
+
end
|
6
|
+
|
7
|
+
def read
|
8
|
+
File.open(@filename, 'r') do |state_file|
|
9
|
+
state_file.gets.chomp.split(':').map(&:to_i)
|
10
|
+
end
|
11
|
+
rescue Errno::ENOENT
|
12
|
+
# if the file doesn't exist, just return the thing
|
13
|
+
[0,0]
|
14
|
+
end
|
15
|
+
|
16
|
+
def write(entry_timestamp, entry_id)
|
17
|
+
File.open(@filename, 'w') do |state_file|
|
18
|
+
state_file.puts("#{entry_timestamp}:#{entry_id}")
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
metadata
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: redis-slowlog-stasher
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Jeremy Kitchen
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-05-14 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: redis
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 3.0.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 3.0.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: bunny
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 1.0.0
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.0.0
|
46
|
+
description: watches the redis slowlog command and sends structured slowlog entries
|
47
|
+
to logstash for processing and storage
|
48
|
+
email: jeremy@nationbuilder.com
|
49
|
+
executables:
|
50
|
+
- redis-slowlog-stasher
|
51
|
+
extensions: []
|
52
|
+
extra_rdoc_files: []
|
53
|
+
files:
|
54
|
+
- lib/redis_slowlog_stasher.rb
|
55
|
+
- lib/redis_slowlog_stasher/statefile.rb
|
56
|
+
- lib/redis_slowlog_stasher/argparser.rb
|
57
|
+
- lib/redis_slowlog_stasher/version.rb
|
58
|
+
- bin/redis-slowlog-stasher
|
59
|
+
- README.md
|
60
|
+
- LICENSE
|
61
|
+
homepage: https://github.com/3dna/redis-slowlog-stasher
|
62
|
+
licenses:
|
63
|
+
- BSD-3-Clause
|
64
|
+
post_install_message:
|
65
|
+
rdoc_options: []
|
66
|
+
require_paths:
|
67
|
+
- lib
|
68
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ! '>='
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
version: '0'
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
none: false
|
76
|
+
requirements:
|
77
|
+
- - ! '>='
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '0'
|
80
|
+
requirements: []
|
81
|
+
rubyforge_project:
|
82
|
+
rubygems_version: 1.8.24
|
83
|
+
signing_key:
|
84
|
+
specification_version: 3
|
85
|
+
summary: put redis slowlog entries into logstash
|
86
|
+
test_files: []
|
87
|
+
has_rdoc:
|