appsignal 0.10.6 → 0.11.0.beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/lib/appsignal.rb +3 -1
- data/lib/appsignal/agent.rb +44 -13
- data/lib/appsignal/aggregator.rb +1 -0
- data/lib/appsignal/event.rb +20 -0
- data/lib/appsignal/event/moped_event.rb +85 -0
- data/lib/appsignal/integrations/resque.rb +4 -5
- data/lib/appsignal/ipc.rb +68 -0
- data/lib/appsignal/params_sanitizer.rb +117 -0
- data/lib/appsignal/transaction.rb +11 -12
- data/lib/appsignal/transaction/params_sanitizer.rb +2 -117
- data/lib/appsignal/transmitter.rb +11 -0
- data/lib/appsignal/version.rb +1 -1
- data/spec/lib/appsignal/agent_spec.rb +114 -50
- data/spec/lib/appsignal/aggregator_spec.rb +1 -0
- data/spec/lib/appsignal/event/moped_event_spec.rb +190 -0
- data/spec/lib/appsignal/event_spec.rb +48 -0
- data/spec/lib/appsignal/ipc_spec.rb +128 -0
- data/spec/lib/appsignal/{transaction/params_sanitizer_spec.rb → params_sanitizer_spec.rb} +2 -2
- data/spec/lib/appsignal/transaction_spec.rb +33 -25
- data/spec/lib/appsignal_spec.rb +1 -1
- data/spec/spec_helper.rb +4 -0
- data/spec/support/helpers/notification_helpers.rb +1 -1
- metadata +16 -9
- data/lib/appsignal/pipe.rb +0 -44
- data/spec/lib/appsignal/pipe_spec.rb +0 -60
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d94bbce0b6f553797da38925a40704be00398b01
|
4
|
+
data.tar.gz: 27e0ba9502c0f57b5842980dcfd9d72f66a4ae78
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aca7235ba46c21dfe16effded54ca808aea9f3faa7e87359374e1caac5ea8570ceb16f8b36454ab42b91a6c6e8eec54bf05471b2ee71ac8462071c6b1ac0eb93
|
7
|
+
data.tar.gz: ffc98ceed5e8343981fa11237a9c9a12350d66d7973b9aa16774c53d5f9a43f166c69bf3152187e4f62b8d7a294954227a5f9d5dc74431c472a0af36dfc880b6
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# 0.11.0
|
2
|
+
* Improved inter process communication
|
3
|
+
* Retry sending data when the push api is not reachable
|
4
|
+
* Our own event handling to allow for more flexibility and reliability
|
5
|
+
when using a threaded environment
|
6
|
+
|
1
7
|
# 0.10.6
|
2
8
|
* Add config option to skip session data
|
3
9
|
|
data/lib/appsignal.rb
CHANGED
@@ -184,6 +184,7 @@ module Appsignal
|
|
184
184
|
end
|
185
185
|
|
186
186
|
require 'appsignal/agent'
|
187
|
+
require 'appsignal/event'
|
187
188
|
require 'appsignal/aggregator'
|
188
189
|
require 'appsignal/aggregator/post_processor'
|
189
190
|
require 'appsignal/aggregator/middleware'
|
@@ -192,10 +193,11 @@ require 'appsignal/config'
|
|
192
193
|
require 'appsignal/marker'
|
193
194
|
require 'appsignal/rack/listener'
|
194
195
|
require 'appsignal/rack/instrumentation'
|
196
|
+
require 'appsignal/params_sanitizer'
|
195
197
|
require 'appsignal/transaction'
|
196
198
|
require 'appsignal/transaction/formatter'
|
197
199
|
require 'appsignal/transaction/params_sanitizer'
|
198
200
|
require 'appsignal/transmitter'
|
199
|
-
require 'appsignal/
|
201
|
+
require 'appsignal/ipc'
|
200
202
|
require 'appsignal/version'
|
201
203
|
require 'appsignal/integrations/rails'
|
data/lib/appsignal/agent.rb
CHANGED
@@ -1,9 +1,10 @@
|
|
1
1
|
module Appsignal
|
2
2
|
class Agent
|
3
3
|
ACTION = 'log_entries'.freeze
|
4
|
+
AGGREGATOR_LIMIT = 5 # Five minutes with a sleep time of 60 seconds
|
4
5
|
|
5
6
|
attr_accessor :aggregator, :thread, :master_pid, :pid, :active, :sleep_time,
|
6
|
-
:transmitter, :subscriber, :paused
|
7
|
+
:transmitter, :subscriber, :paused, :aggregator_queue
|
7
8
|
|
8
9
|
def initialize
|
9
10
|
return unless Appsignal.config.active?
|
@@ -12,10 +13,12 @@ module Appsignal
|
|
12
13
|
else
|
13
14
|
@sleep_time = 60.0
|
14
15
|
end
|
15
|
-
@master_pid
|
16
|
-
@pid
|
17
|
-
@aggregator
|
18
|
-
@transmitter
|
16
|
+
@master_pid = Process.pid
|
17
|
+
@pid = @master_pid
|
18
|
+
@aggregator = Aggregator.new
|
19
|
+
@transmitter = Transmitter.new(ACTION)
|
20
|
+
@aggregator_queue = []
|
21
|
+
|
19
22
|
subscribe
|
20
23
|
start_thread
|
21
24
|
@active = true
|
@@ -33,6 +36,7 @@ module Appsignal
|
|
33
36
|
sleep(rand(sleep_time))
|
34
37
|
loop do
|
35
38
|
send_queue if aggregator.has_transactions?
|
39
|
+
truncate_aggregator_queue
|
36
40
|
Appsignal.logger.debug("Sleeping #{sleep_time}")
|
37
41
|
sleep(sleep_time)
|
38
42
|
end
|
@@ -61,7 +65,7 @@ module Appsignal
|
|
61
65
|
# Subscribe to notifications that don't start with a !
|
62
66
|
@subscriber = ActiveSupport::Notifications.subscribe(/^[^!]/) do |*args|
|
63
67
|
if Appsignal::Transaction.current
|
64
|
-
event =
|
68
|
+
event = Appsignal::Event.event_for_instrumentation(*args)
|
65
69
|
if event.name.start_with?('process_action')
|
66
70
|
Appsignal::Transaction.current.set_process_action_event(event)
|
67
71
|
elsif event.name.start_with?('perform_job')
|
@@ -104,18 +108,38 @@ module Appsignal
|
|
104
108
|
end
|
105
109
|
|
106
110
|
begin
|
107
|
-
|
108
|
-
|
109
|
-
transmitter.transmit(aggregator_to_be_sent.post_processed_queue!)
|
110
|
-
)
|
111
|
-
rescue OpenSSL::SSL::SSLError => ex
|
112
|
-
Appsignal.logger.error "OpenSSL::SSL::SSLError: #{ex.message}"
|
111
|
+
add_to_aggregator_queue(aggregator_to_be_sent.post_processed_queue!)
|
112
|
+
send_aggregators
|
113
113
|
rescue Exception => ex
|
114
114
|
Appsignal.logger.error "#{ex.class} while sending queue: #{ex.message}"
|
115
115
|
Appsignal.logger.error ex.backtrace.join('\n')
|
116
116
|
end
|
117
117
|
end
|
118
118
|
|
119
|
+
def add_to_aggregator_queue(aggregator)
|
120
|
+
@aggregator_queue.unshift(aggregator)
|
121
|
+
end
|
122
|
+
|
123
|
+
def send_aggregators
|
124
|
+
@aggregator_queue.map! do |payload|
|
125
|
+
begin
|
126
|
+
if handle_result(transmitter.transmit(payload))
|
127
|
+
nil
|
128
|
+
else
|
129
|
+
payload
|
130
|
+
end
|
131
|
+
rescue *Transmitter::HTTP_ERRORS
|
132
|
+
payload
|
133
|
+
end
|
134
|
+
end.compact!
|
135
|
+
end
|
136
|
+
|
137
|
+
def truncate_aggregator_queue(limit = AGGREGATOR_LIMIT)
|
138
|
+
return unless @aggregator_queue.length > limit
|
139
|
+
Appsignal.logger.error "Aggregator queue to large, removing items"
|
140
|
+
@aggregator_queue = @aggregator_queue.first(limit)
|
141
|
+
end
|
142
|
+
|
119
143
|
def clear_queue
|
120
144
|
Appsignal.logger.debug('Clearing queue')
|
121
145
|
# Replace aggregator while making sure no thread
|
@@ -150,27 +174,34 @@ module Appsignal
|
|
150
174
|
Appsignal.logger.debug "Queue sent, response code: #{code}"
|
151
175
|
case code.to_i
|
152
176
|
when 200 # ok
|
177
|
+
true
|
153
178
|
when 420 # Enhance Your Calm
|
154
179
|
Appsignal.logger.info 'Increasing sleep time since the server told us to'
|
155
180
|
@sleep_time = sleep_time * 1.5
|
181
|
+
true
|
156
182
|
when 413 # Request Entity Too Large
|
157
183
|
Appsignal.logger.info 'Decreasing sleep time since our last push was too large'
|
158
184
|
@sleep_time = sleep_time / 1.5
|
185
|
+
true
|
159
186
|
when 429
|
160
187
|
Appsignal.logger.error 'Too many requests sent'
|
161
188
|
shutdown(false, 429)
|
189
|
+
true
|
162
190
|
when 406
|
163
191
|
Appsignal.logger.error 'Your appsignal gem cannot communicate with the API anymore, please upgrade.'
|
164
192
|
shutdown(false, 406)
|
193
|
+
true
|
165
194
|
when 402
|
166
195
|
Appsignal.logger.error 'Payment required'
|
167
196
|
shutdown(false, 402)
|
197
|
+
true
|
168
198
|
when 401
|
169
199
|
Appsignal.logger.error 'API token cannot be authorized'
|
170
200
|
shutdown(false, 401)
|
201
|
+
true
|
171
202
|
else
|
172
203
|
Appsignal.logger.error "Unknown Appsignal response code: '#{code}'"
|
173
|
-
|
204
|
+
false
|
174
205
|
end
|
175
206
|
end
|
176
207
|
end
|
data/lib/appsignal/aggregator.rb
CHANGED
@@ -0,0 +1,20 @@
|
|
1
|
+
class Appsignal::Event < ActiveSupport::Notifications::Event
|
2
|
+
def sanitize!
|
3
|
+
@payload = Appsignal::ParamsSanitizer.sanitize(@payload)
|
4
|
+
end
|
5
|
+
|
6
|
+
def truncate!
|
7
|
+
@payload = {}
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.event_for_instrumentation(*args)
|
11
|
+
case args[0]
|
12
|
+
when 'query.moped'
|
13
|
+
Appsignal::Event::MopedEvent.new(*args)
|
14
|
+
else
|
15
|
+
new(*args)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
require 'appsignal/event/moped_event'
|
@@ -0,0 +1,85 @@
|
|
1
|
+
class Appsignal::Event::MopedEvent < Appsignal::Event
|
2
|
+
def initialize(name, start, ending, transaction_id, payload)
|
3
|
+
super(name, start, ending, transaction_id, transform_payload(payload))
|
4
|
+
end
|
5
|
+
|
6
|
+
def transform_payload(payload)
|
7
|
+
if payload[:ops] && payload[:ops].length > 0
|
8
|
+
transformed_ops = [].tap do |arr|
|
9
|
+
payload[:ops].each do |op|
|
10
|
+
arr << payload_from_op(op.dup)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
payload[:ops] = transformed_ops
|
14
|
+
end
|
15
|
+
payload
|
16
|
+
end
|
17
|
+
|
18
|
+
def payload_from_op(payload)
|
19
|
+
case payload.class.to_s
|
20
|
+
when 'Moped::Protocol::Command'
|
21
|
+
{
|
22
|
+
:type => 'Command',
|
23
|
+
:database => payload.full_collection_name,
|
24
|
+
:selector => sanitize(payload.selector)
|
25
|
+
}
|
26
|
+
when 'Moped::Protocol::Query'
|
27
|
+
{
|
28
|
+
:type => 'Query',
|
29
|
+
:database => payload.full_collection_name,
|
30
|
+
:selector => sanitize(payload.selector),
|
31
|
+
:flags => payload.flags,
|
32
|
+
:limit => payload.limit,
|
33
|
+
:skip => payload.skip,
|
34
|
+
:fields => payload.fields
|
35
|
+
}
|
36
|
+
when 'Moped::Protocol::Delete'
|
37
|
+
{
|
38
|
+
:type => 'Delete',
|
39
|
+
:database => payload.full_collection_name,
|
40
|
+
:selector => sanitize(payload.selector),
|
41
|
+
:flags => payload.flags,
|
42
|
+
}
|
43
|
+
when 'Moped::Protocol::Insert'
|
44
|
+
{
|
45
|
+
:type => 'Insert',
|
46
|
+
:database => payload.full_collection_name,
|
47
|
+
:documents => sanitize(payload.documents),
|
48
|
+
:flags => payload.flags,
|
49
|
+
}
|
50
|
+
when 'Moped::Protocol::Update'
|
51
|
+
{
|
52
|
+
:type => 'Update',
|
53
|
+
:database => payload.full_collection_name,
|
54
|
+
:selector => sanitize(payload.selector),
|
55
|
+
:update => sanitize(payload.update),
|
56
|
+
:flags => payload.flags,
|
57
|
+
}
|
58
|
+
else
|
59
|
+
{
|
60
|
+
:type => payload.class.to_s.gsub('Moped::Protocol::', ''),
|
61
|
+
:database => payload.full_collection_name
|
62
|
+
}
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def sanitize(params)
|
67
|
+
if params.is_a?(Hash)
|
68
|
+
{}.tap do |hsh|
|
69
|
+
params.each do |key, val|
|
70
|
+
hsh[key] = sanitize(val)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
elsif params.is_a?(Array)
|
74
|
+
if params.first.is_a?(String)
|
75
|
+
['?']
|
76
|
+
else
|
77
|
+
params.map do |item|
|
78
|
+
sanitize(item)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
else
|
82
|
+
'?'
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
@@ -17,16 +17,15 @@ if defined?(::Resque)
|
|
17
17
|
end
|
18
18
|
end
|
19
19
|
|
20
|
-
#
|
20
|
+
# Set up IPC
|
21
21
|
Resque.before_first_fork do
|
22
|
-
Appsignal::
|
22
|
+
Appsignal::IPC::Server.start
|
23
23
|
end
|
24
24
|
|
25
25
|
# In the fork, stop the normal agent startup
|
26
|
-
# and stop listening to the
|
26
|
+
# and stop listening to the server
|
27
27
|
Resque.after_fork do |job|
|
28
|
-
Appsignal.
|
29
|
-
Appsignal::Pipe.current.stop_listening!
|
28
|
+
Appsignal::IPC.forked!
|
30
29
|
end
|
31
30
|
|
32
31
|
# Extend the default job class with AppSignal instrumentation
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'drb/drb'
|
2
|
+
|
3
|
+
module Appsignal
|
4
|
+
class IPC
|
5
|
+
class << self
|
6
|
+
def forked!
|
7
|
+
Server.stop
|
8
|
+
Client.start
|
9
|
+
Appsignal.agent.stop_thread
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class Server
|
14
|
+
class << self
|
15
|
+
attr_reader :uri
|
16
|
+
|
17
|
+
def start
|
18
|
+
local_tmp_path = File.join(Appsignal.config.root_path, 'tmp')
|
19
|
+
if File.exists?(local_tmp_path)
|
20
|
+
@uri = 'drbunix:' + File.join(local_tmp_path, "appsignal-#{Process.pid}")
|
21
|
+
else
|
22
|
+
@uri = "drbunix:/tmp/appsignal-#{Process.pid}"
|
23
|
+
end
|
24
|
+
|
25
|
+
Appsignal.logger.info("Starting IPC server, listening on #{uri}")
|
26
|
+
DRb.start_service(uri, Appsignal::IPC::Server)
|
27
|
+
end
|
28
|
+
|
29
|
+
def stop
|
30
|
+
Appsignal.logger.debug('Stopping IPC server')
|
31
|
+
DRb.stop_service
|
32
|
+
end
|
33
|
+
|
34
|
+
def enqueue(transaction)
|
35
|
+
Appsignal.logger.debug("Receiving transaction #{transaction.request_id} in IPC server")
|
36
|
+
Appsignal.enqueue(transaction)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class Client
|
42
|
+
class << self
|
43
|
+
attr_reader :server
|
44
|
+
|
45
|
+
def start
|
46
|
+
Appsignal.logger.debug('Starting IPC client')
|
47
|
+
@server = DRbObject.new_with_uri(Appsignal::IPC::Server.uri)
|
48
|
+
@active = true
|
49
|
+
end
|
50
|
+
|
51
|
+
def stop
|
52
|
+
Appsignal.logger.debug('Stopping IPC client')
|
53
|
+
@server = nil
|
54
|
+
@active = false
|
55
|
+
end
|
56
|
+
|
57
|
+
def enqueue(transaction)
|
58
|
+
Appsignal.logger.debug("Sending transaction #{transaction.request_id} in IPC client")
|
59
|
+
@server.enqueue(transaction)
|
60
|
+
end
|
61
|
+
|
62
|
+
def active?
|
63
|
+
!! @active
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,117 @@
|
|
1
|
+
module Appsignal
|
2
|
+
class ParamsSanitizer
|
3
|
+
class << self
|
4
|
+
def sanitize(params)
|
5
|
+
ParamsSanitizerCopy.sanitize_value(params)
|
6
|
+
end
|
7
|
+
|
8
|
+
def sanitize!(params)
|
9
|
+
ParamsSanitizerDestructive.sanitize_value(params)
|
10
|
+
end
|
11
|
+
|
12
|
+
def scrub(params)
|
13
|
+
ParamsSanitizerCopyScrub.sanitize_value(params)
|
14
|
+
end
|
15
|
+
|
16
|
+
def scrub!(params)
|
17
|
+
ParamsSanitizerDestructiveScrub.sanitize_value(params)
|
18
|
+
end
|
19
|
+
|
20
|
+
protected
|
21
|
+
|
22
|
+
def sanitize_value(value)
|
23
|
+
case value
|
24
|
+
when Hash
|
25
|
+
sanitize_hash(value)
|
26
|
+
when Array
|
27
|
+
sanitize_array(value)
|
28
|
+
when Fixnum, String, Symbol
|
29
|
+
unmodified(value)
|
30
|
+
else
|
31
|
+
inspected(value)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def sanitize_hash_with_target(source_hash, target_hash)
|
36
|
+
source_hash.each_pair do |key, value|
|
37
|
+
target_hash[key] = sanitize_value(value)
|
38
|
+
end
|
39
|
+
target_hash
|
40
|
+
end
|
41
|
+
|
42
|
+
def sanitize_array_with_target(source_array, target_array)
|
43
|
+
source_array.each_with_index do |item, index|
|
44
|
+
target_array[index] = sanitize_value(item)
|
45
|
+
end
|
46
|
+
target_array
|
47
|
+
end
|
48
|
+
|
49
|
+
def unmodified(value)
|
50
|
+
value
|
51
|
+
end
|
52
|
+
|
53
|
+
def inspected(value)
|
54
|
+
value.inspect
|
55
|
+
rescue
|
56
|
+
# It turns out that sometimes inspect can fail
|
57
|
+
"#<#{value.class.to_s}/>"
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
class ParamsSanitizerCopy < ParamsSanitizer
|
63
|
+
class << self
|
64
|
+
protected
|
65
|
+
|
66
|
+
def sanitize_hash(hash)
|
67
|
+
sanitize_hash_with_target(hash, {})
|
68
|
+
end
|
69
|
+
|
70
|
+
def sanitize_array(array)
|
71
|
+
sanitize_array_with_target(array, [])
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
class ParamsSanitizerDestructive < ParamsSanitizer
|
77
|
+
class << self
|
78
|
+
protected
|
79
|
+
|
80
|
+
def sanitize_hash(hash)
|
81
|
+
sanitize_hash_with_target(hash, hash)
|
82
|
+
end
|
83
|
+
|
84
|
+
def sanitize_array(array)
|
85
|
+
sanitize_array_with_target(array, array)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
class ParamsSanitizerCopyScrub < ParamsSanitizerCopy
|
91
|
+
class << self
|
92
|
+
protected
|
93
|
+
|
94
|
+
def unmodified(value)
|
95
|
+
'?'
|
96
|
+
end
|
97
|
+
|
98
|
+
def inspected(value)
|
99
|
+
'?'
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
class ParamsSanitizerDestructiveScrub < ParamsSanitizerDestructive
|
105
|
+
class << self
|
106
|
+
protected
|
107
|
+
|
108
|
+
def unmodified(value)
|
109
|
+
'?'
|
110
|
+
end
|
111
|
+
|
112
|
+
def inspected(value)
|
113
|
+
'?'
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|