appsignal 0.10.6 → 0.11.0.beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 29ac953458214ed9268bd6d52afd40b5c45785cd
4
- data.tar.gz: 858cfc04cd727f92934d36c6d1dbcda240404d85
3
+ metadata.gz: d94bbce0b6f553797da38925a40704be00398b01
4
+ data.tar.gz: 27e0ba9502c0f57b5842980dcfd9d72f66a4ae78
5
5
  SHA512:
6
- metadata.gz: 8e70b79af43d968f5b18e5d526ffc79a4f31e1e7f147f828b47cd011dbcb4455b9ca3eeadfb9262628e1bc4be3a83cbd550a48c20f5ee245eea7dec264ab6132
7
- data.tar.gz: 8e453b41199aef1fa956a701b3d8a929c36d6d9222e5f216106e929e15e76510b66d6be543943796c652ab6235b1aa27006216717ca32bccad548e493fffbc0c
6
+ metadata.gz: aca7235ba46c21dfe16effded54ca808aea9f3faa7e87359374e1caac5ea8570ceb16f8b36454ab42b91a6c6e8eec54bf05471b2ee71ac8462071c6b1ac0eb93
7
+ data.tar.gz: ffc98ceed5e8343981fa11237a9c9a12350d66d7973b9aa16774c53d5f9a43f166c69bf3152187e4f62b8d7a294954227a5f9d5dc74431c472a0af36dfc880b6
@@ -1,3 +1,9 @@
1
+ # 0.11.0
2
+ * Improved inter process communication
3
+ * Retry sending data when the push api is not reachable
4
+ * Our own event handling to allow for more flexibility and reliability
5
+ when using a threaded environment
6
+
1
7
  # 0.10.6
2
8
  * Add config option to skip session data
3
9
 
@@ -184,6 +184,7 @@ module Appsignal
184
184
  end
185
185
 
186
186
  require 'appsignal/agent'
187
+ require 'appsignal/event'
187
188
  require 'appsignal/aggregator'
188
189
  require 'appsignal/aggregator/post_processor'
189
190
  require 'appsignal/aggregator/middleware'
@@ -192,10 +193,11 @@ require 'appsignal/config'
192
193
  require 'appsignal/marker'
193
194
  require 'appsignal/rack/listener'
194
195
  require 'appsignal/rack/instrumentation'
196
+ require 'appsignal/params_sanitizer'
195
197
  require 'appsignal/transaction'
196
198
  require 'appsignal/transaction/formatter'
197
199
  require 'appsignal/transaction/params_sanitizer'
198
200
  require 'appsignal/transmitter'
199
- require 'appsignal/pipe'
201
+ require 'appsignal/ipc'
200
202
  require 'appsignal/version'
201
203
  require 'appsignal/integrations/rails'
@@ -1,9 +1,10 @@
1
1
  module Appsignal
2
2
  class Agent
3
3
  ACTION = 'log_entries'.freeze
4
+ AGGREGATOR_LIMIT = 5 # Five minutes with a sleep time of 60 seconds
4
5
 
5
6
  attr_accessor :aggregator, :thread, :master_pid, :pid, :active, :sleep_time,
6
- :transmitter, :subscriber, :paused
7
+ :transmitter, :subscriber, :paused, :aggregator_queue
7
8
 
8
9
  def initialize
9
10
  return unless Appsignal.config.active?
@@ -12,10 +13,12 @@ module Appsignal
12
13
  else
13
14
  @sleep_time = 60.0
14
15
  end
15
- @master_pid = Process.pid
16
- @pid = @master_pid
17
- @aggregator = Aggregator.new
18
- @transmitter = Transmitter.new(ACTION)
16
+ @master_pid = Process.pid
17
+ @pid = @master_pid
18
+ @aggregator = Aggregator.new
19
+ @transmitter = Transmitter.new(ACTION)
20
+ @aggregator_queue = []
21
+
19
22
  subscribe
20
23
  start_thread
21
24
  @active = true
@@ -33,6 +36,7 @@ module Appsignal
33
36
  sleep(rand(sleep_time))
34
37
  loop do
35
38
  send_queue if aggregator.has_transactions?
39
+ truncate_aggregator_queue
36
40
  Appsignal.logger.debug("Sleeping #{sleep_time}")
37
41
  sleep(sleep_time)
38
42
  end
@@ -61,7 +65,7 @@ module Appsignal
61
65
  # Subscribe to notifications that don't start with a !
62
66
  @subscriber = ActiveSupport::Notifications.subscribe(/^[^!]/) do |*args|
63
67
  if Appsignal::Transaction.current
64
- event = ActiveSupport::Notifications::Event.new(*args)
68
+ event = Appsignal::Event.event_for_instrumentation(*args)
65
69
  if event.name.start_with?('process_action')
66
70
  Appsignal::Transaction.current.set_process_action_event(event)
67
71
  elsif event.name.start_with?('perform_job')
@@ -104,18 +108,38 @@ module Appsignal
104
108
  end
105
109
 
106
110
  begin
107
- return unless aggregator_to_be_sent.has_transactions?
108
- handle_result(
109
- transmitter.transmit(aggregator_to_be_sent.post_processed_queue!)
110
- )
111
- rescue OpenSSL::SSL::SSLError => ex
112
- Appsignal.logger.error "OpenSSL::SSL::SSLError: #{ex.message}"
111
+ add_to_aggregator_queue(aggregator_to_be_sent.post_processed_queue!)
112
+ send_aggregators
113
113
  rescue Exception => ex
114
114
  Appsignal.logger.error "#{ex.class} while sending queue: #{ex.message}"
115
115
  Appsignal.logger.error ex.backtrace.join('\n')
116
116
  end
117
117
  end
118
118
 
119
+ def add_to_aggregator_queue(aggregator)
120
+ @aggregator_queue.unshift(aggregator)
121
+ end
122
+
123
+ def send_aggregators
124
+ @aggregator_queue.map! do |payload|
125
+ begin
126
+ if handle_result(transmitter.transmit(payload))
127
+ nil
128
+ else
129
+ payload
130
+ end
131
+ rescue *Transmitter::HTTP_ERRORS
132
+ payload
133
+ end
134
+ end.compact!
135
+ end
136
+
137
+ def truncate_aggregator_queue(limit = AGGREGATOR_LIMIT)
138
+ return unless @aggregator_queue.length > limit
139
+ Appsignal.logger.error "Aggregator queue to large, removing items"
140
+ @aggregator_queue = @aggregator_queue.first(limit)
141
+ end
142
+
119
143
  def clear_queue
120
144
  Appsignal.logger.debug('Clearing queue')
121
145
  # Replace aggregator while making sure no thread
@@ -150,27 +174,34 @@ module Appsignal
150
174
  Appsignal.logger.debug "Queue sent, response code: #{code}"
151
175
  case code.to_i
152
176
  when 200 # ok
177
+ true
153
178
  when 420 # Enhance Your Calm
154
179
  Appsignal.logger.info 'Increasing sleep time since the server told us to'
155
180
  @sleep_time = sleep_time * 1.5
181
+ true
156
182
  when 413 # Request Entity Too Large
157
183
  Appsignal.logger.info 'Decreasing sleep time since our last push was too large'
158
184
  @sleep_time = sleep_time / 1.5
185
+ true
159
186
  when 429
160
187
  Appsignal.logger.error 'Too many requests sent'
161
188
  shutdown(false, 429)
189
+ true
162
190
  when 406
163
191
  Appsignal.logger.error 'Your appsignal gem cannot communicate with the API anymore, please upgrade.'
164
192
  shutdown(false, 406)
193
+ true
165
194
  when 402
166
195
  Appsignal.logger.error 'Payment required'
167
196
  shutdown(false, 402)
197
+ true
168
198
  when 401
169
199
  Appsignal.logger.error 'API token cannot be authorized'
170
200
  shutdown(false, 401)
201
+ true
171
202
  else
172
203
  Appsignal.logger.error "Unknown Appsignal response code: '#{code}'"
173
- clear_queue
204
+ false
174
205
  end
175
206
  end
176
207
  end
@@ -19,6 +19,7 @@ module Appsignal
19
19
  when :slow_request
20
20
  pre_process_slowness!(transaction)
21
21
  when :exception
22
+ transaction.clear_events!
22
23
  transaction.convert_values_to_primitives!
23
24
  end
24
25
  counts[transaction.type] += 1
@@ -0,0 +1,20 @@
1
+ class Appsignal::Event < ActiveSupport::Notifications::Event
2
+ def sanitize!
3
+ @payload = Appsignal::ParamsSanitizer.sanitize(@payload)
4
+ end
5
+
6
+ def truncate!
7
+ @payload = {}
8
+ end
9
+
10
+ def self.event_for_instrumentation(*args)
11
+ case args[0]
12
+ when 'query.moped'
13
+ Appsignal::Event::MopedEvent.new(*args)
14
+ else
15
+ new(*args)
16
+ end
17
+ end
18
+ end
19
+
20
+ require 'appsignal/event/moped_event'
@@ -0,0 +1,85 @@
1
+ class Appsignal::Event::MopedEvent < Appsignal::Event
2
+ def initialize(name, start, ending, transaction_id, payload)
3
+ super(name, start, ending, transaction_id, transform_payload(payload))
4
+ end
5
+
6
+ def transform_payload(payload)
7
+ if payload[:ops] && payload[:ops].length > 0
8
+ transformed_ops = [].tap do |arr|
9
+ payload[:ops].each do |op|
10
+ arr << payload_from_op(op.dup)
11
+ end
12
+ end
13
+ payload[:ops] = transformed_ops
14
+ end
15
+ payload
16
+ end
17
+
18
+ def payload_from_op(payload)
19
+ case payload.class.to_s
20
+ when 'Moped::Protocol::Command'
21
+ {
22
+ :type => 'Command',
23
+ :database => payload.full_collection_name,
24
+ :selector => sanitize(payload.selector)
25
+ }
26
+ when 'Moped::Protocol::Query'
27
+ {
28
+ :type => 'Query',
29
+ :database => payload.full_collection_name,
30
+ :selector => sanitize(payload.selector),
31
+ :flags => payload.flags,
32
+ :limit => payload.limit,
33
+ :skip => payload.skip,
34
+ :fields => payload.fields
35
+ }
36
+ when 'Moped::Protocol::Delete'
37
+ {
38
+ :type => 'Delete',
39
+ :database => payload.full_collection_name,
40
+ :selector => sanitize(payload.selector),
41
+ :flags => payload.flags,
42
+ }
43
+ when 'Moped::Protocol::Insert'
44
+ {
45
+ :type => 'Insert',
46
+ :database => payload.full_collection_name,
47
+ :documents => sanitize(payload.documents),
48
+ :flags => payload.flags,
49
+ }
50
+ when 'Moped::Protocol::Update'
51
+ {
52
+ :type => 'Update',
53
+ :database => payload.full_collection_name,
54
+ :selector => sanitize(payload.selector),
55
+ :update => sanitize(payload.update),
56
+ :flags => payload.flags,
57
+ }
58
+ else
59
+ {
60
+ :type => payload.class.to_s.gsub('Moped::Protocol::', ''),
61
+ :database => payload.full_collection_name
62
+ }
63
+ end
64
+ end
65
+
66
+ def sanitize(params)
67
+ if params.is_a?(Hash)
68
+ {}.tap do |hsh|
69
+ params.each do |key, val|
70
+ hsh[key] = sanitize(val)
71
+ end
72
+ end
73
+ elsif params.is_a?(Array)
74
+ if params.first.is_a?(String)
75
+ ['?']
76
+ else
77
+ params.map do |item|
78
+ sanitize(item)
79
+ end
80
+ end
81
+ else
82
+ '?'
83
+ end
84
+ end
85
+ end
@@ -17,16 +17,15 @@ if defined?(::Resque)
17
17
  end
18
18
  end
19
19
 
20
- # Create a pipe for the workers to write to
20
+ # Set up IPC
21
21
  Resque.before_first_fork do
22
- Appsignal::Pipe.init
22
+ Appsignal::IPC::Server.start
23
23
  end
24
24
 
25
25
  # In the fork, stop the normal agent startup
26
- # and stop listening to the pipe (we'll only use it for writing)
26
+ # and stop listening to the server
27
27
  Resque.after_fork do |job|
28
- Appsignal.agent.stop_thread
29
- Appsignal::Pipe.current.stop_listening!
28
+ Appsignal::IPC.forked!
30
29
  end
31
30
 
32
31
  # Extend the default job class with AppSignal instrumentation
@@ -0,0 +1,68 @@
1
+ require 'drb/drb'
2
+
3
+ module Appsignal
4
+ class IPC
5
+ class << self
6
+ def forked!
7
+ Server.stop
8
+ Client.start
9
+ Appsignal.agent.stop_thread
10
+ end
11
+ end
12
+
13
+ class Server
14
+ class << self
15
+ attr_reader :uri
16
+
17
+ def start
18
+ local_tmp_path = File.join(Appsignal.config.root_path, 'tmp')
19
+ if File.exists?(local_tmp_path)
20
+ @uri = 'drbunix:' + File.join(local_tmp_path, "appsignal-#{Process.pid}")
21
+ else
22
+ @uri = "drbunix:/tmp/appsignal-#{Process.pid}"
23
+ end
24
+
25
+ Appsignal.logger.info("Starting IPC server, listening on #{uri}")
26
+ DRb.start_service(uri, Appsignal::IPC::Server)
27
+ end
28
+
29
+ def stop
30
+ Appsignal.logger.debug('Stopping IPC server')
31
+ DRb.stop_service
32
+ end
33
+
34
+ def enqueue(transaction)
35
+ Appsignal.logger.debug("Receiving transaction #{transaction.request_id} in IPC server")
36
+ Appsignal.enqueue(transaction)
37
+ end
38
+ end
39
+ end
40
+
41
+ class Client
42
+ class << self
43
+ attr_reader :server
44
+
45
+ def start
46
+ Appsignal.logger.debug('Starting IPC client')
47
+ @server = DRbObject.new_with_uri(Appsignal::IPC::Server.uri)
48
+ @active = true
49
+ end
50
+
51
+ def stop
52
+ Appsignal.logger.debug('Stopping IPC client')
53
+ @server = nil
54
+ @active = false
55
+ end
56
+
57
+ def enqueue(transaction)
58
+ Appsignal.logger.debug("Sending transaction #{transaction.request_id} in IPC client")
59
+ @server.enqueue(transaction)
60
+ end
61
+
62
+ def active?
63
+ !! @active
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,117 @@
1
+ module Appsignal
2
+ class ParamsSanitizer
3
+ class << self
4
+ def sanitize(params)
5
+ ParamsSanitizerCopy.sanitize_value(params)
6
+ end
7
+
8
+ def sanitize!(params)
9
+ ParamsSanitizerDestructive.sanitize_value(params)
10
+ end
11
+
12
+ def scrub(params)
13
+ ParamsSanitizerCopyScrub.sanitize_value(params)
14
+ end
15
+
16
+ def scrub!(params)
17
+ ParamsSanitizerDestructiveScrub.sanitize_value(params)
18
+ end
19
+
20
+ protected
21
+
22
+ def sanitize_value(value)
23
+ case value
24
+ when Hash
25
+ sanitize_hash(value)
26
+ when Array
27
+ sanitize_array(value)
28
+ when Fixnum, String, Symbol
29
+ unmodified(value)
30
+ else
31
+ inspected(value)
32
+ end
33
+ end
34
+
35
+ def sanitize_hash_with_target(source_hash, target_hash)
36
+ source_hash.each_pair do |key, value|
37
+ target_hash[key] = sanitize_value(value)
38
+ end
39
+ target_hash
40
+ end
41
+
42
+ def sanitize_array_with_target(source_array, target_array)
43
+ source_array.each_with_index do |item, index|
44
+ target_array[index] = sanitize_value(item)
45
+ end
46
+ target_array
47
+ end
48
+
49
+ def unmodified(value)
50
+ value
51
+ end
52
+
53
+ def inspected(value)
54
+ value.inspect
55
+ rescue
56
+ # It turns out that sometimes inspect can fail
57
+ "#<#{value.class.to_s}/>"
58
+ end
59
+ end
60
+ end
61
+
62
+ class ParamsSanitizerCopy < ParamsSanitizer
63
+ class << self
64
+ protected
65
+
66
+ def sanitize_hash(hash)
67
+ sanitize_hash_with_target(hash, {})
68
+ end
69
+
70
+ def sanitize_array(array)
71
+ sanitize_array_with_target(array, [])
72
+ end
73
+ end
74
+ end
75
+
76
+ class ParamsSanitizerDestructive < ParamsSanitizer
77
+ class << self
78
+ protected
79
+
80
+ def sanitize_hash(hash)
81
+ sanitize_hash_with_target(hash, hash)
82
+ end
83
+
84
+ def sanitize_array(array)
85
+ sanitize_array_with_target(array, array)
86
+ end
87
+ end
88
+ end
89
+
90
+ class ParamsSanitizerCopyScrub < ParamsSanitizerCopy
91
+ class << self
92
+ protected
93
+
94
+ def unmodified(value)
95
+ '?'
96
+ end
97
+
98
+ def inspected(value)
99
+ '?'
100
+ end
101
+ end
102
+ end
103
+
104
+ class ParamsSanitizerDestructiveScrub < ParamsSanitizerDestructive
105
+ class << self
106
+ protected
107
+
108
+ def unmodified(value)
109
+ '?'
110
+ end
111
+
112
+ def inspected(value)
113
+ '?'
114
+ end
115
+ end
116
+ end
117
+ end