logjam_agent 0.29.5 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +49 -28
- data/Rakefile +5 -0
- data/lib/logjam_agent.rb +13 -4
- data/lib/logjam_agent/active_support/core_ext/array/extract.rb +21 -0
- data/lib/logjam_agent/active_support/parameter_filter.rb +128 -0
- data/lib/logjam_agent/buffered_logger.rb +17 -42
- data/lib/logjam_agent/middleware.rb +16 -4
- data/lib/logjam_agent/rack/logger.rb +19 -81
- data/lib/logjam_agent/rack/rails_support.rb +26 -0
- data/lib/logjam_agent/rack/sinatra_request.rb +32 -0
- data/lib/logjam_agent/railtie.rb +3 -0
- data/lib/logjam_agent/receiver.rb +22 -0
- data/lib/logjam_agent/sinatra.rb +116 -0
- data/lib/logjam_agent/syslog_like_formatter.rb +15 -7
- data/lib/logjam_agent/util.rb +6 -1
- data/lib/logjam_agent/version.rb +1 -1
- data/lib/logjam_agent/zmq_forwarder.rb +55 -30
- data/test/sinatra_app.rb +32 -0
- data/test/sinatra_classic_app.rb +31 -0
- data/test/sinatra_classic_test.rb +20 -0
- data/test/sinatra_test.rb +54 -0
- data/test/test_helper.rb +8 -3
- data/test/util_test.rb +5 -0
- metadata +69 -16
- data/.gitignore +0 -5
- data/Gemfile +0 -4
- data/logjam_agent.gemspec +0 -34
- data/script/console +0 -28
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b5c6ecbf3ed8bc61db07e66fb6d5626e1734e6765781a29ad4bbafb261ccf890
|
4
|
+
data.tar.gz: 67a2ef0e834278c2241aafceac7805a72457a1cb68348a48bb0ec95f99658924
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '08137b0a31ce7054b3965ffea84c267d45e08752234f93ad4e2777ac28a0e4954dc4bf0f9dd2e6f1fd02f8979fd5a39a28e5dfa00138f4a6ca8f3a0ac21ba354'
|
7
|
+
data.tar.gz: e0405de81c09f70eecfe77842d25bd150c0fe6a13f028c0cdcbc7d01c029ec786da1b722774a6c4a162112dd1dda04bed0de918469810f3da965241f7d38e322
|
data/README.md
CHANGED
@@ -5,9 +5,14 @@ Client side library for logjam.
|
|
5
5
|
Hooks into Rails, collects log lines, performance metrics, error/exception infomation and Rack
|
6
6
|
environment information and sends this data to [Logjam](https://github.com/skaes/logjam_app).
|
7
7
|
|
8
|
+
Has experimental support for Sinatra.
|
9
|
+
|
8
10
|
Currently only one mechanism is available for data transport:
|
9
11
|
ZeroMQ. Support for AMQP has been dropped.
|
10
12
|
|
13
|
+
[](https://travis-ci.org/github/skaes/logjam_agent)
|
14
|
+
|
15
|
+
|
11
16
|
## Usage
|
12
17
|
|
13
18
|
For ZeroMQ, add
|
@@ -110,10 +115,8 @@ end
|
|
110
115
|
|
111
116
|
### Generating unique request ids
|
112
117
|
|
113
|
-
The agent generates unique request ids for all request handled
|
114
|
-
|
115
|
-
available in the application. Otherwise it will fall back to use the
|
116
|
-
standard `SecureRandom` class shipped with Ruby.
|
118
|
+
The agent generates unique request ids for all request handled using standard
|
119
|
+
`SecureRandom` class shipped with Ruby.
|
117
120
|
|
118
121
|
### Generating JSON
|
119
122
|
|
@@ -121,38 +124,56 @@ The agent will try to use the [Oj](https://github.com/ohler55/oj) to
|
|
121
124
|
generate JSON. If this is not available in your application, it will
|
122
125
|
fall back to the `to_json` method.
|
123
126
|
|
124
|
-
## Troubleshooting
|
125
127
|
|
126
|
-
|
127
|
-
`logjam_agent_error.log` which you can find under `Rails.root/log`.
|
128
|
-
If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
|
128
|
+
### Sinatra
|
129
129
|
|
130
|
-
|
130
|
+
Supports both classic and modular Sinatra applications. Since Sinatra doesn't have built
|
131
|
+
in action names like Rails, you'll have to declare them in your handlers, or in a before
|
132
|
+
filter. Example:
|
131
133
|
|
132
134
|
```ruby
|
133
|
-
|
135
|
+
require 'logjam_agent/sinatra'
|
136
|
+
|
137
|
+
use LogjamAgent::Sinatra::Middleware
|
138
|
+
|
139
|
+
class SinatraTestApp < Sinatra::Base
|
140
|
+
register LogjamAgent::Sinatra
|
141
|
+
|
142
|
+
configure do
|
143
|
+
set :loglevel, :debug
|
144
|
+
setup_logjam_logger
|
145
|
+
|
146
|
+
LogjamAgent.application_name = "myapp"
|
147
|
+
LogjamAgent.add_forwarder(:zmq, :host => "my-logjam-broker")
|
148
|
+
LogjamAgent.parameter_filters << :password
|
149
|
+
end
|
150
|
+
|
151
|
+
before '/index' do
|
152
|
+
action_name "Simple#index"
|
153
|
+
end
|
154
|
+
|
155
|
+
get '/index' do
|
156
|
+
logger.info 'Hello World!'
|
157
|
+
'Hello World!'
|
158
|
+
end
|
159
|
+
end
|
134
160
|
```
|
135
161
|
|
136
|
-
|
162
|
+
The environment name is picked up from either the environment variable `LOGJAM_ENV`, or
|
163
|
+
Sinatra's environment setting.
|
137
164
|
|
138
|
-
|
165
|
+
Set the environment variable `APP_LOG_TO_STDOUT` if you want to log to `STDOUT`.
|
166
|
+
Otherwise, logs will appear in the subdirectory `log` of your application's root.
|
139
167
|
|
140
|
-
Copyright (c) 2013 - 2019 Stefan Kaes
|
141
168
|
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
furnished to do so, subject to the following conditions:
|
169
|
+
## Troubleshooting
|
170
|
+
|
171
|
+
If the agent experiences problems when sending data, it will log information to a file named
|
172
|
+
`logjam_agent_error.log` which you can find under `Rails.root/log`.
|
173
|
+
If you set the `RAILS_LOG_TO_STDOUT` environment variable, those logs will be available through `stderr`.
|
148
174
|
|
149
|
-
|
150
|
-
all copies or substantial portions of the Software.
|
175
|
+
This behavior is customizable via a module level call back method:
|
151
176
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
156
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
157
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
158
|
-
THE SOFTWARE.
|
177
|
+
```ruby
|
178
|
+
LogjamAgent.error_handler = lambda {|exception| ... }
|
179
|
+
```
|
data/Rakefile
CHANGED
@@ -7,8 +7,13 @@ Rake::TestTask.new do |t|
|
|
7
7
|
t.libs << "test"
|
8
8
|
t.test_files = FileList['test/**/*_test.rb']
|
9
9
|
t.verbose = true
|
10
|
+
t.ruby_opts = %w(-W1)
|
10
11
|
end
|
11
12
|
|
12
13
|
task :default do
|
13
14
|
Rake::Task[:test].invoke
|
14
15
|
end
|
16
|
+
|
17
|
+
task :integration do
|
18
|
+
sh "cd railsapp && rake"
|
19
|
+
end
|
data/lib/logjam_agent.rb
CHANGED
@@ -36,9 +36,7 @@ require "logjam_agent/request"
|
|
36
36
|
require "logjam_agent/buffered_logger"
|
37
37
|
require "logjam_agent/syslog_like_formatter"
|
38
38
|
|
39
|
-
if defined?(Rails)
|
40
|
-
require "logjam_agent/railtie"
|
41
|
-
end
|
39
|
+
require "logjam_agent/railtie" if defined?(Rails::Railtie)
|
42
40
|
|
43
41
|
# monkey patch log levels to include NONE
|
44
42
|
require 'logger'
|
@@ -74,6 +72,9 @@ module LogjamAgent
|
|
74
72
|
mattr_accessor :action_name_proc
|
75
73
|
self.action_name_proc = lambda{|name| name}
|
76
74
|
|
75
|
+
mattr_accessor :parameter_filters
|
76
|
+
self.parameter_filters = []
|
77
|
+
|
77
78
|
def self.get_hostname
|
78
79
|
n = Socket.gethostname
|
79
80
|
if n.split('.').size > 1
|
@@ -97,6 +98,9 @@ module LogjamAgent
|
|
97
98
|
mattr_accessor :disabled
|
98
99
|
self.disabled = false
|
99
100
|
|
101
|
+
mattr_accessor :ensure_ping_at_exit
|
102
|
+
self.ensure_ping_at_exit = true
|
103
|
+
|
100
104
|
mattr_accessor :obfuscate_ips
|
101
105
|
self.obfuscate_ips = false
|
102
106
|
|
@@ -109,7 +113,12 @@ module LogjamAgent
|
|
109
113
|
self.obfuscated_cookies = [/_session\z/]
|
110
114
|
|
111
115
|
def self.cookie_obfuscator
|
112
|
-
@cookie_obfuscator ||=
|
116
|
+
@cookie_obfuscator ||=
|
117
|
+
if defined?(ActiveSupport::ParameterFilter)
|
118
|
+
ActiveSupport::ParameterFilter.new(obfuscated_cookies)
|
119
|
+
else
|
120
|
+
ActionDispatch::Http::ParameterFilter.new(obfuscated_cookies)
|
121
|
+
end
|
113
122
|
end
|
114
123
|
|
115
124
|
extend RequestHandling
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class Array
|
4
|
+
# Removes and returns the elements for which the block returns a true value.
|
5
|
+
# If no block is given, an Enumerator is returned instead.
|
6
|
+
#
|
7
|
+
# numbers = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
|
8
|
+
# odd_numbers = numbers.extract! { |number| number.odd? } # => [1, 3, 5, 7, 9]
|
9
|
+
# numbers # => [0, 2, 4, 6, 8]
|
10
|
+
def extract!
|
11
|
+
return to_enum(:extract!) { size } unless block_given?
|
12
|
+
|
13
|
+
extracted_elements = []
|
14
|
+
|
15
|
+
reject! do |element|
|
16
|
+
extracted_elements << element if yield(element)
|
17
|
+
end
|
18
|
+
|
19
|
+
extracted_elements
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/core_ext/object/duplicable"
|
4
|
+
require_relative "core_ext/array/extract"
|
5
|
+
|
6
|
+
module ActiveSupport
|
7
|
+
# +ParameterFilter+ allows you to specify keys for sensitive data from
|
8
|
+
# hash-like object and replace corresponding value. Filtering only certain
|
9
|
+
# sub-keys from a hash is possible by using the dot notation:
|
10
|
+
# 'credit_card.number'. If a proc is given, each key and value of a hash and
|
11
|
+
# all sub-hashes are passed to it, where the value or the key can be replaced
|
12
|
+
# using String#replace or similar methods.
|
13
|
+
#
|
14
|
+
# ActiveSupport::ParameterFilter.new([:password])
|
15
|
+
# => replaces the value to all keys matching /password/i with "[FILTERED]"
|
16
|
+
#
|
17
|
+
# ActiveSupport::ParameterFilter.new([:foo, "bar"])
|
18
|
+
# => replaces the value to all keys matching /foo|bar/i with "[FILTERED]"
|
19
|
+
#
|
20
|
+
# ActiveSupport::ParameterFilter.new(["credit_card.code"])
|
21
|
+
# => replaces { credit_card: {code: "xxxx"} } with "[FILTERED]", does not
|
22
|
+
# change { file: { code: "xxxx"} }
|
23
|
+
#
|
24
|
+
# ActiveSupport::ParameterFilter.new([-> (k, v) do
|
25
|
+
# v.reverse! if k =~ /secret/i
|
26
|
+
# end])
|
27
|
+
# => reverses the value to all keys matching /secret/i
|
28
|
+
class ParameterFilter
|
29
|
+
FILTERED = "[FILTERED]" # :nodoc:
|
30
|
+
|
31
|
+
# Create instance with given filters. Supported type of filters are +String+, +Regexp+, and +Proc+.
|
32
|
+
# Other types of filters are treated as +String+ using +to_s+.
|
33
|
+
# For +Proc+ filters, key, value, and optional original hash is passed to block arguments.
|
34
|
+
#
|
35
|
+
# ==== Options
|
36
|
+
#
|
37
|
+
# * <tt>:mask</tt> - A replaced object when filtered. Defaults to +"[FILTERED]"+
|
38
|
+
def initialize(filters = [], mask: FILTERED)
|
39
|
+
@filters = filters
|
40
|
+
@mask = mask
|
41
|
+
end
|
42
|
+
|
43
|
+
# Mask value of +params+ if key matches one of filters.
|
44
|
+
def filter(params)
|
45
|
+
compiled_filter.call(params)
|
46
|
+
end
|
47
|
+
|
48
|
+
# Returns filtered value for given key. For +Proc+ filters, third block argument is not populated.
|
49
|
+
def filter_param(key, value)
|
50
|
+
@filters.empty? ? value : compiled_filter.value_for_key(key, value)
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
def compiled_filter
|
55
|
+
@compiled_filter ||= CompiledFilter.compile(@filters, mask: @mask)
|
56
|
+
end
|
57
|
+
|
58
|
+
class CompiledFilter # :nodoc:
|
59
|
+
def self.compile(filters, mask:)
|
60
|
+
return lambda { |params| params.dup } if filters.empty?
|
61
|
+
|
62
|
+
strings, regexps, blocks = [], [], []
|
63
|
+
|
64
|
+
filters.each do |item|
|
65
|
+
case item
|
66
|
+
when Proc
|
67
|
+
blocks << item
|
68
|
+
when Regexp
|
69
|
+
regexps << item
|
70
|
+
else
|
71
|
+
strings << Regexp.escape(item.to_s)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
deep_regexps = regexps.extract! { |r| r.to_s.include?("\\.") }
|
76
|
+
deep_strings = strings.extract! { |s| s.include?("\\.") }
|
77
|
+
|
78
|
+
regexps << Regexp.new(strings.join("|"), true) unless strings.empty?
|
79
|
+
deep_regexps << Regexp.new(deep_strings.join("|"), true) unless deep_strings.empty?
|
80
|
+
|
81
|
+
new regexps, deep_regexps, blocks, mask: mask
|
82
|
+
end
|
83
|
+
|
84
|
+
attr_reader :regexps, :deep_regexps, :blocks
|
85
|
+
|
86
|
+
def initialize(regexps, deep_regexps, blocks, mask:)
|
87
|
+
@regexps = regexps
|
88
|
+
@deep_regexps = deep_regexps.any? ? deep_regexps : nil
|
89
|
+
@blocks = blocks
|
90
|
+
@mask = mask
|
91
|
+
end
|
92
|
+
|
93
|
+
def call(params, parents = [], original_params = params)
|
94
|
+
filtered_params = params.class.new
|
95
|
+
|
96
|
+
params.each do |key, value|
|
97
|
+
filtered_params[key] = value_for_key(key, value, parents, original_params)
|
98
|
+
end
|
99
|
+
|
100
|
+
filtered_params
|
101
|
+
end
|
102
|
+
|
103
|
+
def value_for_key(key, value, parents = [], original_params = nil)
|
104
|
+
parents.push(key) if deep_regexps
|
105
|
+
if regexps.any? { |r| r.match?(key.to_s) }
|
106
|
+
value = @mask
|
107
|
+
elsif deep_regexps && (joined = parents.join(".")) && deep_regexps.any? { |r| r.match?(joined) }
|
108
|
+
value = @mask
|
109
|
+
elsif value.is_a?(Hash)
|
110
|
+
value = call(value, parents, original_params)
|
111
|
+
elsif value.is_a?(Array)
|
112
|
+
# If we don't pop the current parent it will be duplicated as we
|
113
|
+
# process each array value.
|
114
|
+
parents.pop if deep_regexps
|
115
|
+
value = value.map { |v| value_for_key(key, v, parents, original_params) }
|
116
|
+
# Restore the parent stack after processing the array.
|
117
|
+
parents.push(key) if deep_regexps
|
118
|
+
elsif blocks.any?
|
119
|
+
key = key.dup if key.duplicable?
|
120
|
+
value = value.dup if value.duplicable?
|
121
|
+
blocks.each { |b| b.arity == 2 ? b.call(key, value) : b.call(key, value, original_params) }
|
122
|
+
end
|
123
|
+
parents.pop if deep_regexps
|
124
|
+
value
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -1,54 +1,31 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
|
3
|
-
|
4
|
-
require 'active_support/buffered_logger'
|
5
|
-
require 'active_support/core_ext/logger'
|
6
|
-
if ActiveSupport::VERSION::STRING >= "3.2"
|
7
|
-
require 'active_support/tagged_logging'
|
8
|
-
# monkey patch to handle exceptions correctly
|
9
|
-
# not needed for rails 4 as this uses a Formatter to add the tags
|
10
|
-
class ActiveSupport::TaggedLogging
|
11
|
-
def initialize(logger)
|
12
|
-
@logger = logger
|
13
|
-
if logger.is_a?(LogjamAgent::BufferedLogger)
|
14
|
-
self.class.class_eval <<-EVAL, __FILE__, __LINE__ + 1
|
15
|
-
def add(severity, message = nil, progname = nil, &block)
|
16
|
-
@logger.add(severity, message, progname, tags_text, &block)
|
17
|
-
end
|
18
|
-
EVAL
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
23
|
-
else
|
24
|
-
require 'active_support/logger'
|
3
|
+
require 'active_support/logger'
|
25
4
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
5
|
+
class LogjamAgent::ConsoleFormatter < Logger::Formatter
|
6
|
+
# This method is invoked when a log event occurs
|
7
|
+
def call(severity, timestamp, progname, msg)
|
8
|
+
"[#{format_time(timestamp)}] #{String === msg ? msg : msg.inspect}\n"
|
9
|
+
end
|
31
10
|
|
32
|
-
|
33
|
-
|
34
|
-
end
|
11
|
+
def format_time(timestamp)
|
12
|
+
timestamp.strftime("%H:%M:%S.#{"%06d" % timestamp.usec}")
|
35
13
|
end
|
14
|
+
end
|
36
15
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
end
|
16
|
+
class ActiveSupport::Logger
|
17
|
+
class << self
|
18
|
+
alias_method :original_broadcast, :broadcast
|
19
|
+
def broadcast(logger)
|
20
|
+
logger.formatter = LogjamAgent::ConsoleFormatter.new
|
21
|
+
logger.formatter.extend(ActiveSupport::TaggedLogging::Formatter)
|
22
|
+
original_broadcast(logger)
|
45
23
|
end
|
46
24
|
end
|
47
25
|
end
|
48
26
|
|
49
27
|
module LogjamAgent
|
50
|
-
class BufferedLogger <
|
51
|
-
ActiveSupport::BufferedLogger : ActiveSupport::Logger )
|
28
|
+
class BufferedLogger < ActiveSupport::Logger
|
52
29
|
|
53
30
|
attr_accessor :formatter
|
54
31
|
|
@@ -57,8 +34,6 @@ module LogjamAgent
|
|
57
34
|
|
58
35
|
def initialize(*args)
|
59
36
|
super(*args)
|
60
|
-
# stupid bug in the buffered logger code (Rails::VERSION::STRING < "3.2")
|
61
|
-
@log.write "\n" if @log && respond_to?(:buffer)
|
62
37
|
@formatter = lambda{|_, _, _, message| message}
|
63
38
|
end
|
64
39
|
|
@@ -1,21 +1,30 @@
|
|
1
1
|
module LogjamAgent
|
2
2
|
class Middleware
|
3
|
-
def initialize(app,
|
3
|
+
def initialize(app, framework = :rails)
|
4
4
|
@app = app
|
5
|
-
@
|
5
|
+
@framework = framework
|
6
|
+
unless %i{rails sinatra}.include?(framework)
|
7
|
+
raise ArgumentError.new("Invalid logjam_agent framework: #{framework}. Only :rails and :sinatra are valid!")
|
8
|
+
end
|
9
|
+
@reraise = defined?(Rails::Railtie) && Rails.env.test?
|
6
10
|
end
|
7
11
|
|
8
12
|
def call(env)
|
13
|
+
env["logjam_agent.framework"] = @framework
|
9
14
|
strip_encoding_from_etag(env)
|
10
15
|
request = start_request(env)
|
11
16
|
result = @app.call(env)
|
12
17
|
result[1] ||= {}
|
13
18
|
result
|
14
19
|
rescue Exception
|
15
|
-
result = [500, {'Content-Type' => 'text/html'}, ["<html><body><h1>500 Internal Server Error</h1>"]]
|
20
|
+
result = [500, {'Content-Type' => 'text/html'}, ["<html><body><h1>500 Internal Server Error</h1></body></html>"]]
|
21
|
+
raise if @reraise
|
16
22
|
ensure
|
17
23
|
headers = result[1]
|
18
24
|
headers["X-Logjam-Request-Id"] = request.id
|
25
|
+
if env["sinatra.static_file"]
|
26
|
+
request.fields[:action] = "Sinatra#static_file"
|
27
|
+
end
|
19
28
|
unless (request_action = request.fields[:action]).blank?
|
20
29
|
headers["X-Logjam-Request-Action"] = request_action
|
21
30
|
end
|
@@ -41,7 +50,10 @@ module LogjamAgent
|
|
41
50
|
env_name = env["logjam_agent.environment_name"] || LogjamAgent.environment_name
|
42
51
|
caller_id = env["HTTP_X_LOGJAM_CALLER_ID"] || ""
|
43
52
|
caller_action = env["HTTP_X_LOGJAM_ACTION"] || ""
|
44
|
-
|
53
|
+
extra_fields = {}
|
54
|
+
extra_fields[:caller_id] = caller_id if caller_id.present?
|
55
|
+
extra_fields[:caller_action] = caller_action if caller_action.present?
|
56
|
+
LogjamAgent.start_request(app_name, env_name, extra_fields)
|
45
57
|
end
|
46
58
|
|
47
59
|
def finish_request(env)
|