logworm_amqp 0.8.9 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +103 -1
- data/Manifest +8 -7
- data/Rakefile +4 -4
- data/bin/lw-compute +66 -0
- data/bin/lw-tail +84 -0
- data/lib/base/db.rb +8 -6
- data/lib/client/logger.rb +93 -0
- data/lib/client/rack.rb +61 -0
- data/lib/client/rails.rb +105 -0
- data/lib/cmd/compute.rb +66 -0
- data/lib/cmd/tail.rb +106 -0
- data/lib/logworm_amqp.rb +61 -0
- data/lib/logworm_utils.rb +4 -0
- data/logworm_amqp.gemspec +20 -16
- metadata +64 -43
- data/README.md +0 -3
- data/spec/base_spec.rb +0 -143
- data/spec/builder_spec.rb +0 -26
- data/spec/config_spec.rb +0 -36
- data/spec/spec.opts +0 -4
- data/spec/spec_helper.rb +0 -8
- data/tests/builder_test.rb +0 -52
data/CHANGELOG
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
v0.9.0
|
2
|
+
Combines base and client into a single gem
|
3
|
+
|
4
|
+
base
|
5
|
+
----
|
1
6
|
v0.8.9
|
2
7
|
Cleaned up pushed stats output.
|
3
8
|
|
@@ -64,4 +69,101 @@ v0.2.0
|
|
64
69
|
removed app/ libraries. Added tail as a utility, lw-tail as a command
|
65
70
|
|
66
71
|
v0.1.0
|
67
|
-
initial version.
|
72
|
+
initial version.
|
73
|
+
|
74
|
+
|
75
|
+
client
|
76
|
+
------
|
77
|
+
v0.8.8
|
78
|
+
Cleaned up pushed stats output.
|
79
|
+
|
80
|
+
v0.8.7
|
81
|
+
Started logging amqp usage statistics to our servers (randomly) for better QA.
|
82
|
+
|
83
|
+
v.0.8.6
|
84
|
+
No longer prints out amqp information when logging and should also self check for amqp url updates incase it was changed.
|
85
|
+
|
86
|
+
v0.8.5
|
87
|
+
Small fix
|
88
|
+
|
89
|
+
v0.8.0
|
90
|
+
Dropped http for amqp for high scalability and reduce latency
|
91
|
+
|
92
|
+
v0.7.2
|
93
|
+
Renamed Rails and Rack configuration commands:
|
94
|
+
lw_disable_request_logging ==> donot_log_requests
|
95
|
+
lw_log_request_headers ==> log_headers
|
96
|
+
lw_enable_dev_logging ==> log_in_development
|
97
|
+
|
98
|
+
Renamed :headers field in web_log to :request_headers
|
99
|
+
|
100
|
+
v0.7.1
|
101
|
+
lw-tail and lw-compute now receive an optional app parameter, for the cases where you want to call command-line tools from a directory other than the app's directory... or when you have more than one Heroku remote/app from the same directory
|
102
|
+
|
103
|
+
v0.7.0
|
104
|
+
Eliminated long logs. Instead, there's now a parameter (log_request_headers in Rack, lw_log_request_headers in Rails) to add
|
105
|
+
request and response headers to the web_log.
|
106
|
+
Cleaned up fields in the web_log table
|
107
|
+
|
108
|
+
Added option to turn off automatic logging of requests (disable_request_logging in Rack, lw_disable_request_logging in Rails)
|
109
|
+
Added option to turn on logging even if in development mode (enable_dev_logging in Rack, lw_enable_dev_logging in Rails)
|
110
|
+
|
111
|
+
Honor filter_parameter_logging switch when used with Rails
|
112
|
+
|
113
|
+
Cleaned up log and flush
|
114
|
+
Cleaned up display of elapsed time for flush
|
115
|
+
Cleaned up call from Rack and Rails, and enforce timeout (1 sec by default)
|
116
|
+
Added unit tests
|
117
|
+
|
118
|
+
v0.6.2
|
119
|
+
Added list_tables command
|
120
|
+
|
121
|
+
v0.6.1
|
122
|
+
Added a bit of documentation
|
123
|
+
|
124
|
+
v0.6.0
|
125
|
+
Add support for querying API: lw_query
|
126
|
+
|
127
|
+
v0.5.6
|
128
|
+
log HEROKU_QUEUE as an integer value
|
129
|
+
|
130
|
+
v0.5.5
|
131
|
+
Added default logging of HEROKU_QUEUE
|
132
|
+
|
133
|
+
v0.5.4
|
134
|
+
Removed obsolete lw-heroku application
|
135
|
+
|
136
|
+
v0.5.3
|
137
|
+
Show nicer error message with tail and compute if the app is not properly configured.
|
138
|
+
|
139
|
+
v0.5.2
|
140
|
+
Fixes minor issues with in Rails logging
|
141
|
+
|
142
|
+
v0.5.1
|
143
|
+
Fixes minor issues with logging statements sent to console
|
144
|
+
|
145
|
+
v0.5.0
|
146
|
+
Supports new base logworm gem, which can work as a Heroku addon
|
147
|
+
Configuration parameters now stored as URL
|
148
|
+
No error if logworm cannot be properly configured -- lw_log simply ignored
|
149
|
+
No logging in development mode
|
150
|
+
|
151
|
+
v0.4.1
|
152
|
+
Show in console time spent communicating with the server.
|
153
|
+
|
154
|
+
v0.4.0 Added support for running on Rails!
|
155
|
+
To use, add config.gem ‘logworm_client’ to environment.rb
|
156
|
+
Then you can configure in ApplicationController, via logs_requests [:short | :long]+
|
157
|
+
Works from views, models, or controllers.
|
158
|
+
|
159
|
+
v0.3.2 added lw_log method --fixed small issue
|
160
|
+
|
161
|
+
v0.3.1 added lw_log method
|
162
|
+
|
163
|
+
v0.3.0 ensure gem and dependencies live in gemcutter
|
164
|
+
|
165
|
+
v0.2.0 log short requests by default in the rack
|
166
|
+
|
167
|
+
v0.1.1 minor changes
|
168
|
+
|
169
|
+
v0.1.0 initial version.
|
data/Manifest
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
CHANGELOG
|
2
2
|
Manifest
|
3
|
-
README.md
|
4
3
|
Rakefile
|
4
|
+
bin/lw-compute
|
5
|
+
bin/lw-tail
|
5
6
|
lib/base/config.rb
|
6
7
|
lib/base/db.rb
|
7
8
|
lib/base/query_builder.rb
|
9
|
+
lib/client/logger.rb
|
10
|
+
lib/client/rack.rb
|
11
|
+
lib/client/rails.rb
|
12
|
+
lib/cmd/compute.rb
|
13
|
+
lib/cmd/tail.rb
|
8
14
|
lib/logworm_amqp.rb
|
9
|
-
|
10
|
-
spec/builder_spec.rb
|
11
|
-
spec/config_spec.rb
|
12
|
-
spec/spec.opts
|
13
|
-
spec/spec_helper.rb
|
14
|
-
tests/builder_test.rb
|
15
|
+
lib/logworm_utils.rb
|
data/Rakefile
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
require 'echoe'
|
2
|
-
Echoe.new('logworm_amqp', '0.
|
3
|
-
p.description = "logworm logging
|
2
|
+
Echoe.new('logworm_amqp', '0.9.0') do |p|
|
3
|
+
p.description = "logworm - logging service"
|
4
4
|
p.url = "http://www.logworm.com"
|
5
5
|
p.author = "Pomelo, LLC"
|
6
6
|
p.email = "schapira@pomelollc.com"
|
7
7
|
p.ignore_pattern = ["tmp/*", "script/*"]
|
8
|
-
p.development_dependencies = ["
|
9
|
-
p.runtime_dependencies = ["memcache-client", "hpricot", "oauth", "heroku"
|
8
|
+
p.development_dependencies = ["json >=1.4.3", "ruby-hmac", "hpricot", "oauth", "heroku"]
|
9
|
+
p.runtime_dependencies = ["json >=1.4.3", "ruby-hmac", "memcache-client", "hpricot", "oauth", "heroku"]
|
10
10
|
end
|
data/bin/lw-compute
ADDED
@@ -0,0 +1,66 @@
|
|
1
|
+
#!/usr/local/bin/ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'logworm_utils'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
options = {
|
8
|
+
:aggregate_group => [],
|
9
|
+
:conditions => [],
|
10
|
+
:start => nil,
|
11
|
+
:end => nil,
|
12
|
+
:debug => false
|
13
|
+
}
|
14
|
+
|
15
|
+
option_parser = OptionParser.new do |opts|
|
16
|
+
opts.banner = "Usage: #{$0} [options] <log table> <function> <field>"
|
17
|
+
|
18
|
+
opts.on("--app app", String, "Specify the Heroku app") do |app|
|
19
|
+
options[:app] = app.strip
|
20
|
+
end
|
21
|
+
|
22
|
+
opts.on("-g group", String, "Specify an aggregation group (e.g, hour(_ts_utc), or response.code)") do |k|
|
23
|
+
options[:aggregate_group] = k
|
24
|
+
end
|
25
|
+
|
26
|
+
opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
|
27
|
+
options[:conditions] << c.strip
|
28
|
+
end
|
29
|
+
|
30
|
+
opts.on("-s starttime", String, "Specify the start time for the query") do |c|
|
31
|
+
options[:start] = c.strip
|
32
|
+
end
|
33
|
+
|
34
|
+
opts.on("-e endtime", String, "Specify the end time for the query") do |c|
|
35
|
+
options[:end] = c.strip
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("-v", "Show debug information") do
|
39
|
+
options[:debug] = true
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on( '-h', '--help', 'Display this screen' ) do
|
43
|
+
puts option_parser.help
|
44
|
+
exit(1)
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
|
49
|
+
# Parse, and get the required <table>
|
50
|
+
option_parser.parse!
|
51
|
+
if ARGV.size == 3
|
52
|
+
table = ARGV[0].strip
|
53
|
+
function = ARGV[1].strip
|
54
|
+
field = ARGV[2].strip
|
55
|
+
elsif ARGV.size == 2
|
56
|
+
table = ARGV[0].strip
|
57
|
+
function = ARGV[1].strip
|
58
|
+
else
|
59
|
+
puts option_parser.help
|
60
|
+
exit(1)
|
61
|
+
end
|
62
|
+
|
63
|
+
# and run
|
64
|
+
LogwormCompute.new(table, function, field, options).run
|
65
|
+
|
66
|
+
|
data/bin/lw-tail
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
#!/usr/local/bin/ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'logworm_utils'
|
5
|
+
require 'optparse'
|
6
|
+
|
7
|
+
def help(option_parser, options)
|
8
|
+
puts option_parser.help
|
9
|
+
puts
|
10
|
+
LogwormTail.list(options)
|
11
|
+
end
|
12
|
+
|
13
|
+
options = {
|
14
|
+
:limit => 200,
|
15
|
+
:loop => false,
|
16
|
+
:frequency => 10,
|
17
|
+
:fields => [],
|
18
|
+
:conditions => [],
|
19
|
+
:start => nil,
|
20
|
+
:end => nil,
|
21
|
+
:debug => false,
|
22
|
+
:fkat => false
|
23
|
+
}
|
24
|
+
|
25
|
+
option_parser = OptionParser.new do |opts|
|
26
|
+
opts.banner = "Usage: #{$0} [options] <log table>"
|
27
|
+
|
28
|
+
opts.on("--app app", String, "Specify the Heroku app") do |app|
|
29
|
+
options[:app] = app.strip
|
30
|
+
end
|
31
|
+
|
32
|
+
opts.on("-f [secs]", Integer,
|
33
|
+
"Continuously check for more data, every [secs] seconds.", "Default: #{options[:frequency]}") do |s|
|
34
|
+
options[:loop] = true
|
35
|
+
options[:frequency] = s if s
|
36
|
+
end
|
37
|
+
|
38
|
+
opts.on("-r limit", Integer, "Specify how many log entries to fetch.", "Default: #{options[:limit]}") do |n|
|
39
|
+
options[:limit] = n
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on("-k fields", String, "Specify a comma-separated list of fields to retrieve.") do |k|
|
43
|
+
options[:fields] = k.split(",").map {|f| f.strip}
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.on("-c condition", String, "Specify a condition to match. May be used multiple times") do |c|
|
47
|
+
options[:conditions] << c.strip
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on("-s starttime", String, "Specify the start time for the query") do |c|
|
51
|
+
options[:start] = c.strip
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on("-e endtime", String, "Specify the end time for the query") do |c|
|
55
|
+
options[:end] = c.strip
|
56
|
+
end
|
57
|
+
|
58
|
+
opts.on("--flat", "Do not expand log entries when printing them") do
|
59
|
+
options[:flat] = true
|
60
|
+
end
|
61
|
+
|
62
|
+
opts.on("-v", "Show debug information") do
|
63
|
+
options[:debug] = true
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.on( '-h', '--help', 'Display this screen' ) do
|
67
|
+
options[:help] = true
|
68
|
+
end
|
69
|
+
|
70
|
+
end
|
71
|
+
|
72
|
+
# Parse, and get the required <table>
|
73
|
+
option_parser.parse!
|
74
|
+
table = ARGV.pop
|
75
|
+
if !table or options[:help]
|
76
|
+
help option_parser, options
|
77
|
+
exit(1)
|
78
|
+
end
|
79
|
+
table.strip!
|
80
|
+
|
81
|
+
# and run
|
82
|
+
t = LogwormTail.new(table, options).run
|
83
|
+
|
84
|
+
|
data/lib/base/db.rb
CHANGED
@@ -31,7 +31,6 @@ module Logworm
|
|
31
31
|
raise ForbiddenAccessException.new("Incorrect URL Format #{url}") unless match and match.size == 6
|
32
32
|
@consumer_key, @consumer_secret, @host, @token, @token_secret = match[1..5]
|
33
33
|
@connection = OAuth::AccessToken.new(OAuth::Consumer.new(@consumer_key, @consumer_secret), @token, @token_secret)
|
34
|
-
reset_stats
|
35
34
|
end
|
36
35
|
|
37
36
|
def self.with_tokens(token, token_secret)
|
@@ -115,11 +114,12 @@ module Logworm
|
|
115
114
|
resp = db_call(:get, "#{host_with_protocol}/amqp_url")
|
116
115
|
@amqp_url = resp["url"]
|
117
116
|
@stats_freq = resp["stats_freq"]
|
118
|
-
$stderr.puts "logworm server acquired: #{@amqp_url}"
|
117
|
+
$stderr.puts "logworm server acquired: #{@amqp_url.gsub(/[^@]+@/, "amqp://")}"
|
119
118
|
Minion.amqp_url = @amqp_url
|
119
|
+
reset_stats
|
120
120
|
rescue
|
121
121
|
@last_attempt = Time.now
|
122
|
-
$stderr.puts "logworm cannot connect to server;
|
122
|
+
$stderr.puts "logworm cannot connect to server; waiting #{RETRY_FREQUENCY} seconds before retry. log entries will be lost"
|
123
123
|
end
|
124
124
|
end
|
125
125
|
!(@amqp_url.nil?)
|
@@ -149,9 +149,11 @@ module Logworm
|
|
149
149
|
end
|
150
150
|
|
151
151
|
def push_stats()
|
152
|
-
|
153
|
-
|
154
|
-
|
152
|
+
avg = sprintf('%.6f', (@total_time / @tock)).to_f
|
153
|
+
len = sprintf('%.2f', (Time.now - @sampling_start)).to_f
|
154
|
+
to_amqp("lw.stats", {:avg => avg , :max => @amqp_max, :min => @amqp_min,
|
155
|
+
:total => @total_time, :count => @tock, :sampling_length => len})
|
156
|
+
$stderr.puts "logworm statistics: #{@tock} messages sent in last #{len} secs. Times: #{avg * 1000}/#{@amqp_min * 1000}/#{@amqp_max * 1000} (avg/min/max msecs)"
|
155
157
|
reset_stats
|
156
158
|
end
|
157
159
|
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Logworm
|
4
|
+
class LogwormException < Exception ; end
|
5
|
+
|
6
|
+
class Logger
|
7
|
+
$lr_queue = []
|
8
|
+
|
9
|
+
###
|
10
|
+
# Use connection to the backend servers specified in environment variable or config file
|
11
|
+
###
|
12
|
+
def self.use_default_db
|
13
|
+
$lw_server = DB.from_config
|
14
|
+
end
|
15
|
+
|
16
|
+
###
|
17
|
+
# Use a connection to a manually specified server
|
18
|
+
###
|
19
|
+
def self.use_db(db)
|
20
|
+
$lw_server = db
|
21
|
+
end
|
22
|
+
|
23
|
+
###
|
24
|
+
# Returns a reference to the current backend server
|
25
|
+
###
|
26
|
+
def self.db
|
27
|
+
$lw_server
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
###
|
32
|
+
# Starts a new cycle: sets a request_id variable, so that all entries (until flush) share that id
|
33
|
+
###
|
34
|
+
def self.start_cycle
|
35
|
+
$request_id = "#{Thread.current.object_id}-#{(Time.now.utc.to_f * 1000).to_i}"
|
36
|
+
end
|
37
|
+
|
38
|
+
###
|
39
|
+
# Record an entry. Not sent to the servers until 'flush' is called
|
40
|
+
#
|
41
|
+
# Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
|
42
|
+
# of a web request, for example.
|
43
|
+
###
|
44
|
+
def self.log(table, values = {})
|
45
|
+
return unless table and (table.is_a? String or table.is_a? Symbol)
|
46
|
+
return unless values.is_a? Hash
|
47
|
+
|
48
|
+
# Turn keys into symbols, delete empty ones, rename conflicting ones
|
49
|
+
kvalues = values.delete_if {|k,v| k.to_s == ""}.map {|k,v| [k.to_sym, v]}
|
50
|
+
kvalues = Hash[*kvalues.flatten]
|
51
|
+
[:_ts, :_ts_utc, :_request_id].each do |k|
|
52
|
+
kvalues["orig_#{k}".to_sym] = kvalues[k] if kvalues.has_key? k
|
53
|
+
end
|
54
|
+
|
55
|
+
# Add information
|
56
|
+
ts = Time.now.utc
|
57
|
+
kvalues[:_ts_utc] = (ts.to_f * 1000).to_i
|
58
|
+
kvalues[:_ts] = ts.strftime("%Y-%m-%dT%H:%M:%SZ")
|
59
|
+
kvalues[:_request_id] = $request_id if $request_id
|
60
|
+
|
61
|
+
# Enqueue
|
62
|
+
$lr_queue << [table, kvalues]
|
63
|
+
end
|
64
|
+
|
65
|
+
###
|
66
|
+
# Sends the entries to the server, if configured
|
67
|
+
# Returns the number of entries send, and the time it takes
|
68
|
+
#
|
69
|
+
# Warning: may raise Exception if there's a problem. It's up to the called to rescue from it in order to continue the processing
|
70
|
+
# of a web request, for example.
|
71
|
+
###
|
72
|
+
def self.flush
|
73
|
+
to_send = $lr_queue.size
|
74
|
+
|
75
|
+
# Return if no entries
|
76
|
+
return [0,0] if to_send == 0
|
77
|
+
|
78
|
+
# Return if no server
|
79
|
+
unless $lw_server
|
80
|
+
$stderr.puts "\t logworm not configured. #{to_send} entries dropped."
|
81
|
+
$lr_queue = []
|
82
|
+
return [0,0]
|
83
|
+
end
|
84
|
+
|
85
|
+
startTime = Time.now
|
86
|
+
$lw_server.batch_log($lr_queue.to_json)
|
87
|
+
$lr_queue = []
|
88
|
+
endTime = Time.now
|
89
|
+
|
90
|
+
[to_send, (endTime - startTime)]
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
data/lib/client/rack.rb
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
require 'rack/request'
|
2
|
+
|
3
|
+
module Logworm
|
4
|
+
class Rack
|
5
|
+
|
6
|
+
def initialize(app, options = {})
|
7
|
+
@app = app
|
8
|
+
|
9
|
+
@log_requests = (options[:donot_log_requests].nil? or options[:donot_log_requests] != true)
|
10
|
+
@log_headers = (options[:log_headers] and options[:log_headers] == true)
|
11
|
+
@dev_logging = (options[:log_in_development] and options[:log_in_development] == true)
|
12
|
+
Logger.use_default_db
|
13
|
+
@timeout = 1
|
14
|
+
end
|
15
|
+
|
16
|
+
def call(env)
|
17
|
+
return @app.call(env) unless (ENV['RACK_ENV'] == 'production' or (ENV['RACK_ENV'] == 'development' and @dev_logging))
|
18
|
+
|
19
|
+
Logger.start_cycle
|
20
|
+
begin
|
21
|
+
startTime = Time.now
|
22
|
+
status, response_headers, body = @app.call(env)
|
23
|
+
appTime = (Time.now - startTime)
|
24
|
+
ensure
|
25
|
+
log_request(env, status, response_headers, appTime)
|
26
|
+
return [status, response_headers, body]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
private
|
31
|
+
def log_request(env, status, response_headers, appTime)
|
32
|
+
method = env['REQUEST_METHOD']
|
33
|
+
path = (env['REQUEST_PATH'].nil? or env['REQUEST_PATH'] == "") ? "/" : env['REQUEST_PATH']
|
34
|
+
ip = env['REMOTE_ADDR']
|
35
|
+
http_headers = env.reject {|k,v| !(k.to_s =~ /^HTTP/) }
|
36
|
+
queue_size = env['HTTP_X_HEROKU_QUEUE_DEPTH'].nil? ? -1 : env['HTTP_X_HEROKU_QUEUE_DEPTH'].to_i
|
37
|
+
|
38
|
+
entry = { :summary => "#{method} #{path} - #{status} #{appTime}",
|
39
|
+
:request_method => method,
|
40
|
+
:request_path => path,
|
41
|
+
:request_ip => ip,
|
42
|
+
:input => ::Rack::Request.new(env).params,
|
43
|
+
:response_status => status,
|
44
|
+
:profiling => appTime,
|
45
|
+
:queue_size => queue_size}
|
46
|
+
entry[:request_headers] = http_headers if @log_headers
|
47
|
+
entry[:response_headers] = response_headers if @log_headers
|
48
|
+
Logger.log(:web_log, entry) if @log_requests
|
49
|
+
|
50
|
+
begin
|
51
|
+
Timeout::timeout(@timeout) {
|
52
|
+
sent, elapsed = Logger.flush
|
53
|
+
}
|
54
|
+
rescue Exception => e
|
55
|
+
# Ignore --nothing we can do. The list of logs may (and most likely will) be preserved for the next request
|
56
|
+
env['rack.errors'].puts("logworm call failed: #{e}")
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|