edouard-clarity 0.9.9
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -0
- data/Manifest.txt +34 -0
- data/PostInstall.txt +8 -0
- data/README.rdoc +85 -0
- data/Rakefile +23 -0
- data/bin/clarity +10 -0
- data/config/config.yml.sample +7 -0
- data/lib/clarity.rb +24 -0
- data/lib/clarity/cli.rb +94 -0
- data/lib/clarity/commands/grep_command_builder.rb +57 -0
- data/lib/clarity/commands/hostname_command_builder.rb +7 -0
- data/lib/clarity/commands/tail_command_builder.rb +27 -0
- data/lib/clarity/grep_renderer.rb +37 -0
- data/lib/clarity/process_tree.rb +23 -0
- data/lib/clarity/renderers/log_renderer.rb +36 -0
- data/lib/clarity/server.rb +139 -0
- data/lib/clarity/server/basic_auth.rb +24 -0
- data/lib/clarity/server/chunk_http.rb +59 -0
- data/lib/clarity/server/mime_types.rb +23 -0
- data/public/images/spinner_big.gif +0 -0
- data/public/javascripts/app.js +154 -0
- data/public/stylesheets/app.css +55 -0
- data/script/console +10 -0
- data/script/destroy +14 -0
- data/script/generate +14 -0
- data/test/commands/grep_command_builder_test.rb +99 -0
- data/test/commands/tail_command_builder_test.rb +42 -0
- data/test/files/logfile.log +17 -0
- data/test/test_helper.rb +4 -0
- data/test/test_string_scanner.rb +16 -0
- data/views/_header.html.erb +5 -0
- data/views/_toolbar.html.erb +64 -0
- data/views/error.html.erb +11 -0
- data/views/index.html.erb +9 -0
- metadata +163 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
PostInstall.txt
|
4
|
+
README.rdoc
|
5
|
+
Rakefile
|
6
|
+
bin/clarity
|
7
|
+
config/config.yml.sample
|
8
|
+
lib/clarity.rb
|
9
|
+
lib/clarity/cli.rb
|
10
|
+
lib/clarity/commands/grep_command_builder.rb
|
11
|
+
lib/clarity/commands/hostname_command_builder.rb
|
12
|
+
lib/clarity/commands/tail_command_builder.rb
|
13
|
+
lib/clarity/grep_renderer.rb
|
14
|
+
lib/clarity/process_tree.rb
|
15
|
+
lib/clarity/renderers/log_renderer.rb
|
16
|
+
lib/clarity/server.rb
|
17
|
+
lib/clarity/server/basic_auth.rb
|
18
|
+
lib/clarity/server/chunk_http.rb
|
19
|
+
lib/clarity/server/mime_types.rb
|
20
|
+
public/images/spinner_big.gif
|
21
|
+
public/javascripts/app.js
|
22
|
+
public/stylesheets/app.css
|
23
|
+
script/console
|
24
|
+
script/destroy
|
25
|
+
script/generate
|
26
|
+
test/commands/grep_command_builder_test.rb
|
27
|
+
test/commands/tail_command_builder_test.rb
|
28
|
+
test/files/logfile.log
|
29
|
+
test/test_helper.rb
|
30
|
+
test/test_string_scanner.rb
|
31
|
+
views/_header.html.erb
|
32
|
+
views/_toolbar.html.erb
|
33
|
+
views/error.html.erb
|
34
|
+
views/index.html.erb
|
data/PostInstall.txt
ADDED
data/README.rdoc
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
= Clarity
|
2
|
+
|
3
|
+
* http://github.com/tobi/clarity
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Clarity - a log search tool
|
8
|
+
By John Tajima & Tobi Lütke
|
9
|
+
|
10
|
+
Clarity is a Splunk like web interface for your server log files. It supports
|
11
|
+
searching (using grep) as well as trailing log files in realtime. It has been written
|
12
|
+
using the event based architecture based on EventMachine and so allows real-time search
|
13
|
+
of very large log files. If you hit the browser Stop button it will also kill
|
14
|
+
the grep / tail utility.
|
15
|
+
|
16
|
+
We wrote Clarity to allow our support staff to use a simple interface to look
|
17
|
+
through the various log files in our server farm. The application was such a
|
18
|
+
big success internally that we decided to release it as open source.
|
19
|
+
|
20
|
+
== SECURITY:
|
21
|
+
|
22
|
+
*Warning*: Clarity takes parameters from URLs and runs them in the shell.
|
23
|
+
This is essentially the most insecure thing imaginable. You have to make absolutley sure
|
24
|
+
that clarity isn't reachable by the outside world. At the very least use --username and
|
25
|
+
--password to put some protection on it.
|
26
|
+
|
27
|
+
== USAGE:
|
28
|
+
|
29
|
+
clarity --username=admin --password=secret --port=8989 /var/log
|
30
|
+
|
31
|
+
== COMMANDLINE:
|
32
|
+
|
33
|
+
Specific options:
|
34
|
+
-f, --config=FILE Config file (yml)
|
35
|
+
-p, --port=PORT Port to listen on
|
36
|
+
-b, --address=ADDRESS Address to bind to (default 0.0.0.0)
|
37
|
+
--include=MASK File mask of logs to add (default: **/*.log*)
|
38
|
+
--user=USER User to run as
|
39
|
+
Password protection:
|
40
|
+
--username=USER Enable httpauth username
|
41
|
+
--password=PASS Enable httpauth password
|
42
|
+
Misc:
|
43
|
+
-h, --help Show this message.
|
44
|
+
|
45
|
+
|
46
|
+
== SCREENSHOT:
|
47
|
+
|
48
|
+
http://img.skitch.com/20091104-je9kq1a2gfr586ia8y246bq4n8.png
|
49
|
+
|
50
|
+
== REQUIREMENTS:
|
51
|
+
|
52
|
+
* eventmachine
|
53
|
+
* eventmachine_httpserver
|
54
|
+
* json
|
55
|
+
|
56
|
+
== INSTALL:
|
57
|
+
|
58
|
+
* sudo gem install clarity
|
59
|
+
|
60
|
+
== LICENSE:
|
61
|
+
|
62
|
+
(The MIT License)
|
63
|
+
|
64
|
+
Copyright (c) 2009 Tobias Lütke
|
65
|
+
|
66
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
67
|
+
a copy of this software and associated documentation files (the
|
68
|
+
'Software'), to deal in the Software without restriction, including
|
69
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
70
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
71
|
+
permit persons to whom the Software is furnished to do so, subject to
|
72
|
+
the following conditions:
|
73
|
+
|
74
|
+
The above copyright notice and this permission notice shall be
|
75
|
+
included in all copies or substantial portions of the Software.
|
76
|
+
|
77
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
78
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
79
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
80
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
81
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
82
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
83
|
+
SOFTWARE OR THE USE OR OT
|
84
|
+
|
85
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
$LOAD_PATH.unshift File.join(File.dirname(__FILE__), "lib")
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
gem 'hoe', '>= 2.1.0'
|
5
|
+
gem 'newgem'
|
6
|
+
|
7
|
+
require 'hoe'
|
8
|
+
require 'clarity'
|
9
|
+
|
10
|
+
Hoe.plugin :newgem
|
11
|
+
|
12
|
+
$hoe = Hoe.spec 'edouard-clarity' do
|
13
|
+
self.developer 'Tobias Lütke', 'tobi@shopify.com'
|
14
|
+
self.developer 'John Tajima', 'john@shopify.com'
|
15
|
+
self.summary = 'Web interface for grep and tail -f'
|
16
|
+
self.post_install_message = 'PostInstall.txt'
|
17
|
+
self.readme_file = 'README.rdoc'
|
18
|
+
self.extra_deps = [['eventmachine','>= 0.12.10'], ['eventmachine_httpserver','>= 0.2.0'], ["json", ">= 1.0.0"]]
|
19
|
+
self.test_globs = ['test/**/*_test.rb']
|
20
|
+
end
|
21
|
+
|
22
|
+
require 'newgem/tasks'
|
23
|
+
Dir['tasks/**/*.rake'].each { |t| load t }
|
data/bin/clarity
ADDED
data/lib/clarity.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__)) unless $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
|
2
|
+
|
3
|
+
begin
|
4
|
+
require File.join(File.dirname(__FILE__), *%w[.. vendor gems environment])
|
5
|
+
rescue LoadError
|
6
|
+
end
|
7
|
+
|
8
|
+
require 'eventmachine'
|
9
|
+
require 'evma_httpserver'
|
10
|
+
require 'json'
|
11
|
+
require 'yaml'
|
12
|
+
require 'base64'
|
13
|
+
require 'clarity/server'
|
14
|
+
require 'clarity/commands/grep_command_builder'
|
15
|
+
require 'clarity/commands/tail_command_builder'
|
16
|
+
require 'clarity/commands/hostname_command_builder'
|
17
|
+
require 'clarity/renderers/log_renderer'
|
18
|
+
|
19
|
+
module Clarity
|
20
|
+
VERSION = '0.9.9'
|
21
|
+
|
22
|
+
Templates = File.dirname(__FILE__) + '/../views'
|
23
|
+
Public = File.dirname(__FILE__) + '/../public'
|
24
|
+
end
|
data/lib/clarity/cli.rb
ADDED
@@ -0,0 +1,94 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
|
3
|
+
module Clarity
|
4
|
+
class CLI
|
5
|
+
def self.execute(stdout, arguments=[])
|
6
|
+
|
7
|
+
options = {
|
8
|
+
:username => nil,
|
9
|
+
:password => nil,
|
10
|
+
:log_files => nil,
|
11
|
+
:port => 8080,
|
12
|
+
:address => "0.0.0.0",
|
13
|
+
:user => nil,
|
14
|
+
:group => nil,
|
15
|
+
:relative_root => nil
|
16
|
+
}
|
17
|
+
|
18
|
+
mandatory_options = %w( )
|
19
|
+
|
20
|
+
ARGV.options do |opts|
|
21
|
+
opts.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] directory"
|
22
|
+
|
23
|
+
opts.separator " "
|
24
|
+
opts.separator "Specific options:"
|
25
|
+
|
26
|
+
opts.on( "-f", "--config=FILE", String, "Config file (yml)" ) do |opt|
|
27
|
+
config = YAML.load_file( opt )
|
28
|
+
config.keys.each do |key|
|
29
|
+
options[key.to_sym] = config[key]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
opts.on( "-p", "--port=PORT", Integer, "Port to listen on" ) do |opt|
|
34
|
+
options[:port] = opt
|
35
|
+
end
|
36
|
+
|
37
|
+
opts.on( "-b", "--address=ADDRESS", String, "Address to bind to (default 0.0.0.0)" ) do |opt|
|
38
|
+
options[:address] = opt
|
39
|
+
end
|
40
|
+
|
41
|
+
opts.on( "-r", "--relative=ROOT", String, "Run under a relative root" ) do |opt|
|
42
|
+
options[:relative_root] = opt
|
43
|
+
end
|
44
|
+
|
45
|
+
opts.on( "--include=MASK", String, "File mask of logs to add (default: **/*.log*)" ) do |opt|
|
46
|
+
options[:log_files] ||= []
|
47
|
+
options[:log_files] << opt
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on( "--user=USER", String, "User to run as" ) do |opt|
|
51
|
+
options[:user] = opt
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.separator " "
|
55
|
+
opts.separator "Password protection:"
|
56
|
+
|
57
|
+
opts.on( "--username=USER", String, "Enable httpauth username" ) do |opt|
|
58
|
+
options[:username] = opt
|
59
|
+
end
|
60
|
+
|
61
|
+
opts.on( "--password=PASS", String, "Enable httpauth password" ) do |opt|
|
62
|
+
options[:password] = opt
|
63
|
+
end
|
64
|
+
|
65
|
+
opts.separator " "
|
66
|
+
opts.separator "Misc:"
|
67
|
+
|
68
|
+
opts.on( "-h", "--help", "Show this message." ) do
|
69
|
+
puts opts
|
70
|
+
exit
|
71
|
+
end
|
72
|
+
|
73
|
+
opts.separator " "
|
74
|
+
|
75
|
+
begin
|
76
|
+
opts.parse!(arguments)
|
77
|
+
|
78
|
+
options[:log_files] ||= ['**/*.log*']
|
79
|
+
|
80
|
+
if arguments.first
|
81
|
+
Dir.chdir(arguments.first)
|
82
|
+
|
83
|
+
::Clarity::Server.run(options)
|
84
|
+
|
85
|
+
else
|
86
|
+
puts opts
|
87
|
+
exit(1)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
class GrepCommandBuilder
|
2
|
+
|
3
|
+
# parameter names
|
4
|
+
TermParameters = ['term1', 'term2', 'term3']
|
5
|
+
FileParameter = 'file'
|
6
|
+
|
7
|
+
attr_accessor :params
|
8
|
+
attr_reader :terms, :filename, :options
|
9
|
+
|
10
|
+
def initialize(params)
|
11
|
+
@params = params
|
12
|
+
@filename = params.fetch(FileParameter)
|
13
|
+
@terms = TermParameters.map {|term| params.fetch(term, nil) }.compact.reject {|term| term.empty? }
|
14
|
+
@options = ""
|
15
|
+
valid?
|
16
|
+
end
|
17
|
+
|
18
|
+
def valid?
|
19
|
+
raise InvalidParameterError, "Log file parameter not supplied" unless filename && !filename.empty?
|
20
|
+
true
|
21
|
+
end
|
22
|
+
|
23
|
+
def command
|
24
|
+
results = []
|
25
|
+
exec_functions.each_with_index do |cmd, index|
|
26
|
+
results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index].to_s)
|
27
|
+
end
|
28
|
+
%[sh -c '#{results.join(" | ")}']
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def exec_functions
|
33
|
+
case File.extname(filename)
|
34
|
+
when '.gz' then gzip_tools
|
35
|
+
when '.bz2' then bzip_tools
|
36
|
+
else default_tools
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
def gzip_tools
|
42
|
+
cat_tool = (ENV["PATH"].split(":").find{|d| File.exists?(File.join(d, "gzcat"))} ? "zcat" : "gzcat")
|
43
|
+
terms.empty? ? ["#{cat_tool} filename"] : ['zgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
|
44
|
+
end
|
45
|
+
|
46
|
+
def bzip_tools
|
47
|
+
terms.empty? ? ['bzcat filename'] : ['bzgrep options -e term filename'] + ['grep options -e term'] * (terms.size-1)
|
48
|
+
end
|
49
|
+
|
50
|
+
def default_tools
|
51
|
+
terms.empty? ? ['cat filename'] : ['grep options -e term filename'] + ['grep options -e term']* (terms.size-1)
|
52
|
+
end
|
53
|
+
|
54
|
+
|
55
|
+
class InvalidParameterError < StandardError; end
|
56
|
+
|
57
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Handles tailing of log files
|
3
|
+
#
|
4
|
+
class TailCommandBuilder < GrepCommandBuilder
|
5
|
+
|
6
|
+
def valid?
|
7
|
+
raise InvalidParameterError, "Log file parameter not supplied or invalid log file" unless filename && !filename.empty?
|
8
|
+
true
|
9
|
+
end
|
10
|
+
|
11
|
+
def command
|
12
|
+
results = []
|
13
|
+
exec_functions.each_with_index do |cmd, index|
|
14
|
+
if index == 0
|
15
|
+
results << cmd.gsub('filename', filename.to_s)
|
16
|
+
else
|
17
|
+
results << cmd.gsub('filename', filename.to_s).gsub('options', options.to_s).gsub('term', terms[index-1].to_s)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
%[sh -c '#{results.join(" | ")}']
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
def default_tools
|
25
|
+
terms.empty? ? ['tail -n 250 -f filename'] : ['tail -n 250 -f filename'] + ['grep options -e term'] * (terms.size)
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Clarity
|
2
|
+
module GrepRenderer
|
3
|
+
attr_accessor :response
|
4
|
+
attr_writer :renderer
|
5
|
+
|
6
|
+
def renderer
|
7
|
+
@renderer ||= LogRenderer.new
|
8
|
+
end
|
9
|
+
|
10
|
+
# once download is complete, send it to client
|
11
|
+
def receive_data(data)
|
12
|
+
@buffer ||= StringScanner.new("")
|
13
|
+
@buffer << data
|
14
|
+
|
15
|
+
while line = @buffer.scan_until(/\n/)
|
16
|
+
response.chunk renderer.render(line)
|
17
|
+
flush
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def flush
|
22
|
+
response.send_chunks
|
23
|
+
end
|
24
|
+
|
25
|
+
def close
|
26
|
+
ProcessTree.kill(get_status.pid)
|
27
|
+
end
|
28
|
+
|
29
|
+
def unbind
|
30
|
+
response.chunk renderer.finalize
|
31
|
+
response.chunk ''
|
32
|
+
close
|
33
|
+
flush
|
34
|
+
puts 'Done'
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module ProcessTree
|
2
|
+
|
3
|
+
def self.kill(ppid)
|
4
|
+
return if ppid.nil?
|
5
|
+
all_pids = [ppid] + child_pids_of(ppid).flatten.uniq.compact
|
6
|
+
all_pids.each do |pid|
|
7
|
+
Process.kill('TERM',pid.to_i) rescue nil
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.child_pids_of(ppid)
|
12
|
+
out = `ps -opid,ppid | grep #{ppid.to_s}`
|
13
|
+
ids = out.split("\n").map {|line| $1 if line =~ /^\s*([0-9]+)\s.*/ }.compact
|
14
|
+
ids.delete(ppid.to_s)
|
15
|
+
if ids.empty?
|
16
|
+
ids
|
17
|
+
else
|
18
|
+
ids << ids.map {|id| child_pids_of(id) }
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
|
23
|
+
end
|