tingyun_rpm 1.1.4.2 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -1
  3. data/Guardfile +10 -0
  4. data/lib/ting_yun/agent.rb +1 -0
  5. data/lib/ting_yun/agent/agent.rb +16 -27
  6. data/lib/ting_yun/agent/collector/error_collector.rb +7 -18
  7. data/lib/ting_yun/agent/collector/error_collector/noticed_error.rb +26 -21
  8. data/lib/ting_yun/agent/collector/middle_ware_collector/cpu_sampler.rb +4 -9
  9. data/lib/ting_yun/agent/collector/sql_sampler.rb +32 -188
  10. data/lib/ting_yun/agent/collector/sql_sampler/slow_sql.rb +47 -0
  11. data/lib/ting_yun/agent/collector/sql_sampler/sql_trace.rb +73 -0
  12. data/lib/ting_yun/agent/collector/sql_sampler/transaction_sql_data.rb +26 -0
  13. data/lib/ting_yun/agent/collector/stats_engine/metric_stats.rb +6 -5
  14. data/lib/ting_yun/agent/collector/stats_engine/stats_hash.rb +2 -2
  15. data/lib/ting_yun/agent/collector/transaction_sampler.rb +23 -159
  16. data/lib/ting_yun/agent/collector/transaction_sampler/class_method.rb +130 -0
  17. data/lib/ting_yun/agent/collector/transaction_sampler/slowest_sample_buffer.rb +1 -1
  18. data/lib/ting_yun/agent/collector/transaction_sampler/transaction_sample_buffer_base.rb +1 -1
  19. data/lib/ting_yun/agent/cross_app/cross_app_monitor.rb +29 -79
  20. data/lib/ting_yun/agent/cross_app/cross_app_tracing.rb +36 -66
  21. data/lib/ting_yun/agent/database.rb +41 -349
  22. data/lib/ting_yun/agent/database/connection_manager.rb +44 -0
  23. data/lib/ting_yun/agent/database/explain_plan_helpers.rb +173 -0
  24. data/lib/ting_yun/agent/database/obfuscator.rb +151 -0
  25. data/lib/ting_yun/agent/database/statement.rb +70 -0
  26. data/lib/ting_yun/agent/event/event_loop.rb +1 -2
  27. data/lib/ting_yun/agent/instance_methods/connect.rb +8 -20
  28. data/lib/ting_yun/agent/instance_methods/container_data_manager.rb +2 -3
  29. data/lib/ting_yun/agent/instance_methods/handle_errors.rb +6 -1
  30. data/lib/ting_yun/agent/instance_methods/start.rb +13 -81
  31. data/lib/ting_yun/agent/transaction.rb +48 -391
  32. data/lib/ting_yun/agent/transaction/apdex.rb +53 -0
  33. data/lib/ting_yun/agent/transaction/attributes.rb +2 -1
  34. data/lib/ting_yun/agent/transaction/class_method.rb +127 -0
  35. data/lib/ting_yun/agent/transaction/exceptions.rb +42 -0
  36. data/lib/ting_yun/agent/transaction/instance_method.rb +139 -0
  37. data/lib/ting_yun/agent/transaction/request_attributes.rb +9 -39
  38. data/lib/ting_yun/agent/transaction/trace.rb +7 -5
  39. data/lib/ting_yun/agent/transaction/trace_node.rb +1 -3
  40. data/lib/ting_yun/agent/transaction/traced_method_stack.rb +2 -3
  41. data/lib/ting_yun/agent/transaction/transaction_sample_builder.rb +6 -1
  42. data/lib/ting_yun/agent/transaction/transaction_state.rb +59 -17
  43. data/lib/ting_yun/agent/transaction/transaction_timings.rb +72 -0
  44. data/lib/ting_yun/configuration.rb +11 -0
  45. data/lib/ting_yun/configuration/default_source.rb +20 -17
  46. data/lib/ting_yun/configuration/manager.rb +50 -21
  47. data/lib/ting_yun/frameworks.rb +1 -0
  48. data/lib/ting_yun/frameworks/rails.rb +15 -0
  49. data/lib/ting_yun/instrumentation/active_record.rb +12 -18
  50. data/lib/ting_yun/instrumentation/middleware_tracing.rb +8 -14
  51. data/lib/ting_yun/instrumentation/mongo.rb +21 -27
  52. data/lib/ting_yun/instrumentation/mongo_command_log_subscriber.rb +7 -3
  53. data/lib/ting_yun/instrumentation/moped.rb +2 -2
  54. data/lib/ting_yun/instrumentation/net.rb +4 -5
  55. data/lib/ting_yun/instrumentation/rack.rb +1 -2
  56. data/lib/ting_yun/instrumentation/rails4/active_record_subscriber.rb +22 -20
  57. data/lib/ting_yun/instrumentation/redis.rb +2 -2
  58. data/lib/ting_yun/instrumentation/support/controller_instrumentation.rb +1 -1
  59. data/lib/ting_yun/instrumentation/support/external_error.rb +19 -16
  60. data/lib/ting_yun/instrumentation/support/javascript_instrumentor.rb +92 -0
  61. data/lib/ting_yun/instrumentation/support/thrift_helper.rb +73 -0
  62. data/lib/ting_yun/instrumentation/thrift.rb +19 -222
  63. data/lib/ting_yun/logger.rb +1 -0
  64. data/lib/ting_yun/logger/agent_logger.rb +11 -67
  65. data/lib/ting_yun/logger/create_logger_helper.rb +72 -0
  66. data/lib/ting_yun/metrics/metric_data.rb +9 -31
  67. data/lib/ting_yun/metrics/metric_spec.rb +11 -0
  68. data/lib/ting_yun/metrics/stats.rb +24 -1
  69. data/lib/ting_yun/middleware/agent_middleware.rb +28 -0
  70. data/lib/ting_yun/middleware/browser_monitoring.rb +111 -0
  71. data/lib/ting_yun/support/coerce.rb +1 -0
  72. data/lib/ting_yun/support/exception.rb +2 -33
  73. data/lib/ting_yun/support/local_environment.rb +7 -7
  74. data/lib/ting_yun/support/serialize/marshaller.rb +7 -25
  75. data/lib/ting_yun/ting_yun_service.rb +12 -9
  76. data/lib/ting_yun/ting_yun_service/connection.rb +3 -0
  77. data/lib/ting_yun/ting_yun_service/http.rb +4 -1
  78. data/lib/ting_yun/ting_yun_service/request.rb +5 -13
  79. data/lib/ting_yun/ting_yun_service/upload_service.rb +5 -7
  80. data/lib/ting_yun/version.rb +3 -5
  81. data/lib/tingyun_rpm.rb +12 -10
  82. data/tingyun_rpm.gemspec +3 -0
  83. metadata +49 -5
  84. data/.DS_Store +0 -0
  85. data/lib/ting_yun/agent/collector/base_sampler.rb +0 -2
@@ -8,7 +8,7 @@ module TingYun
8
8
  class TransactionSampler
9
9
  class SlowestSampleBuffer < TransactionSampleBufferBase
10
10
 
11
- CAPACITY = 20
11
+ CAPACITY = 500
12
12
 
13
13
  def capacity
14
14
  CAPACITY
@@ -7,7 +7,7 @@ module TingYun
7
7
  class TransactionSampler
8
8
  class TransactionSampleBufferBase
9
9
 
10
- SINGLE_BUFFER_MAX = 20
10
+ SINGLE_BUFFER_MAX = 1000
11
11
  NO_SAMPLES = [].freeze
12
12
 
13
13
  def initialize
@@ -19,113 +19,63 @@ module TingYun
19
19
  register_event_listeners(events)
20
20
  end
21
21
 
22
+
22
23
  # Expected sequence of events:
23
24
  # :before_call will save our cross application request id to the thread
24
25
  # :after_call will write our response headers/metrics and clean up the thread
25
26
  def register_event_listeners(events)
26
27
  TingYun::Agent.logger.debug("Wiring up Cross Application Tracing to events after finished configuring")
27
28
 
28
- events.subscribe(:before_call) do |env| #THREAD_LOCAL_ACCESS
29
- if cross_app_enabled?
29
+ events.subscribe(:cross_app_before_call) do |env| #THREAD_LOCAL_ACCESS
30
+ if TingYun::Agent::CrossAppTracing.cross_app_enabled?
30
31
  state = TingYun::Agent::TransactionState.tl_get
31
- save_referring_transaction_info(state, env) unless env[TY_ID_HEADER].nil?
32
+ state.save_referring_transaction_info(env[TY_ID_HEADER].split(';')) if env[TY_ID_HEADER]
32
33
  end
33
34
  end
34
35
 
35
- events.subscribe(:after_call) do |_status_code, headers, _body| #THREAD_LOCAL_ACCESS
36
- state = TingYun::Agent::TransactionState.tl_get
37
- state.queue_duration = state.current_transaction.queue_time * 1000
38
- state.web_duration = (Time.now - state.current_transaction.start_time) * 1000
39
- insert_response_header(state, headers)
36
+ events.subscribe(:cross_app_after_call) do |_status_code, headers, _body| #THREAD_LOCAL_ACCESS
37
+ insert_response_header(headers) if TingYun::Agent::CrossAppTracing.cross_app_enabled?
40
38
  end
41
39
 
42
40
  end
43
41
 
44
42
 
45
- def cross_app_enabled?
46
- TingYun::Agent::CrossAppTracing.cross_app_enabled?
47
- end
48
-
49
-
50
- def save_referring_transaction_info(state,request)
51
-
52
- info = request[TY_ID_HEADER].split(';')
53
- tingyun_id_secret = info[0]
54
- client_transaction_id = info.find do |e|
55
- e.match(/x=/)
56
- end.split('=')[1] rescue nil
57
- client_req_id = info.find do |e|
58
- e.match(/r=/)
59
- end.split('=')[1] rescue nil
60
-
61
- state.client_tingyun_id_secret = tingyun_id_secret
62
- state.client_transaction_id = client_transaction_id
63
- state.transaction_sample_builder.trace.tx_id = client_transaction_id
64
- state.client_req_id = client_req_id
65
- end
66
-
67
-
68
- def insert_response_header(state, response_headers)
69
- if same_account?(state)
43
+ def insert_response_header(response_headers)
44
+ state = TingYun::Agent::TransactionState.tl_get
45
+ if state.same_account?
70
46
  txn = state.current_transaction
71
- unless txn.nil?
72
- set_response_headers(state, response_headers)
47
+ if txn
48
+ # set_response_headers
49
+ response_headers[TY_DATA_HEADER] = TingYun::Support::Serialize::JSONWrapper.dump build_payload(state)
50
+ TingYun::Agent.logger.debug("now,cross app will send response_headers #{response_headers[TY_DATA_HEADER]}")
73
51
  end
74
- clear_client_tingyun_id_secret(state)
75
- end
76
- end
77
-
78
- def clear_client_tingyun_id_secret(state)
79
- state.client_tingyun_id_secret = nil
80
- end
81
-
82
- def same_account?(state)
83
- server_info = TingYun::Agent.config[:tingyunIdSecret].split('|')
84
- client_info = (state.client_tingyun_id_secret || '').split('|')
85
- if !server_info[0].nil? && server_info[0] == client_info[0] && !server_info[0].empty?
86
- return true
87
- else
88
- return false
89
52
  end
90
53
  end
91
54
 
92
- def set_response_headers(state, response_headers)
93
- response_headers[TY_DATA_HEADER] = TingYun::Support::Serialize::JSONWrapper.dump build_payload(state)
94
- TingYun::Agent.logger.debug("now,cross app will send response_headers #{response_headers[TY_DATA_HEADER]}")
95
- end
96
55
 
97
56
  def build_payload(state)
57
+ timings = state.timings
58
+
98
59
  payload = {
99
- :id => TingYun::Agent.config[:tingyunIdSecret].split('|')[1],
100
- :action => state.current_transaction.best_name,
101
- :trId => state.transaction_sample_builder.trace.guid,
102
- :time => {
103
- :duration => state.web_duration,
104
- :qu => state.queue_duration,
105
- :db => state.sql_duration,
106
- :ex => state.external_duration,
107
- :rds => state.rds_duration,
108
- :mc => state.mc_duration,
109
- :mon => state.mon_duration,
110
- :code => execute_duration(state)
111
- }
60
+ :id => TingYun::Agent.config[:tingyunIdSecret].split('|')[1],
61
+ :action => state.transaction_name,
62
+ :trId => state.trace_id,
63
+ :time => {
64
+ :duration => timings.app_time_in_millis,
65
+ :qu => timings.queue_time_in_millis,
66
+ :db => timings.sql_duration,
67
+ :ex => timings.external_duration,
68
+ :rds => timings.rds_duration,
69
+ :mc => timings.mc_duration,
70
+ :mon => timings.mon_duration,
71
+ :code => timings.app_execute_duration
72
+ }
112
73
  }
113
- payload[:tr] = 1 if slow_action_tracer?(state)
74
+ payload[:tr] = 1 if timings.slow_action_tracer?
114
75
  payload[:r] = state.client_req_id unless state.client_req_id.nil?
115
76
  payload
116
77
  end
117
78
 
118
- def slow_action_tracer?(state)
119
- if state.web_duration > TingYun::Agent.config[:'nbs.action_tracer.action_threshold']
120
- return true
121
- else
122
- return false
123
- end
124
- end
125
-
126
- def execute_duration(state)
127
- state.web_duration - state.queue_duration - state.sql_duration - state.external_duration - state.rds_duration - state.mc_duration - state.mon_duration
128
- end
129
79
  end
130
80
  end
131
81
  end
@@ -5,6 +5,7 @@ require 'ting_yun/agent/transaction'
5
5
  require 'ting_yun/support/http_clients/uri_util'
6
6
  require 'ting_yun/support/serialize/json_wrapper'
7
7
  require 'ting_yun/instrumentation/support/external_error'
8
+ require 'ting_yun/agent/collector/transaction_sampler'
8
9
 
9
10
 
10
11
  module TingYun
@@ -22,26 +23,22 @@ module TingYun
22
23
  TY_DATA_HEADER = 'X-Tingyun-Tx-Data'.freeze
23
24
 
24
25
 
25
-
26
-
27
26
  module_function
28
27
 
29
28
 
30
-
31
29
  def tl_trace_http_request(request)
32
- t0 = Time.now.to_f
33
30
  state = TingYun::Agent::TransactionState.tl_get
34
31
  return yield unless state.execution_traced?
35
32
  return yield unless state.current_transaction #如果还没有创建Transaction,就发生跨应用,就直接先跳过跟踪。
36
33
 
34
+ t0 = Time.now.to_f
37
35
  begin
38
36
  node = start_trace(state, t0, request)
39
37
  response = yield
40
- capture_exception(response,request,'net%2Fhttp')
38
+ capture_exception(response, request, 'net%2Fhttp')
41
39
  rescue => e
42
40
  klass = "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
43
- handle_error(e,klass)
44
- raise e
41
+ handle_error(e, klass)
45
42
  ensure
46
43
  finish_trace(state, t0, node, request, response)
47
44
  end
@@ -51,7 +48,7 @@ module TingYun
51
48
  def start_trace(state, t0, request)
52
49
  inject_request_headers(state, request) if cross_app_enabled?
53
50
  stack = state.traced_method_stack
54
- node = stack.push_frame(state,:http_request,t0)
51
+ node = stack.push_frame(state, :http_request, t0)
55
52
 
56
53
  return node
57
54
  end
@@ -59,49 +56,55 @@ module TingYun
59
56
  def finish_trace(state, t0, node, request, response)
60
57
  t1 = Time.now.to_f
61
58
  duration = (t1- t0) * 1000
62
- state.external_duration = duration
59
+ state.timings.external_duration = duration
63
60
 
64
61
  begin
65
62
  if request
66
- metrics = metrics_for(state, request, response)
63
+ cross_app = response_is_cross_app?(response)
64
+
65
+ metrics = metrics_for(request, response, cross_app)
67
66
  node_name = metrics.pop
68
67
  scoped_metric = metrics.pop
69
68
 
70
- stats_engine.record_scoped_and_unscoped_metrics(state, scoped_metric, metrics, duration)
69
+ ::TingYun::Agent.instance.stats_engine.record_scoped_and_unscoped_metrics(state, scoped_metric, metrics, duration)
71
70
 
72
71
  if node
73
72
  node.name = node_name
74
- add_transaction_trace_info(request, response)
73
+ add_transaction_trace_info(request, response, cross_app)
75
74
  end
76
75
  end
76
+ rescue => err
77
+ TingYun::Agent.logger.error "Uncaught exception while finishing an HTTP request trace", err
77
78
  ensure
78
79
  if node
79
80
  stack = state.traced_method_stack
80
81
  stack.pop_frame(state, node, node_name, t1)
81
82
  end
82
83
  end
83
- rescue => err
84
- TingYun::Agent.logger.error "Uncaught exception while finishing an HTTP request trace", err
85
- raise
86
84
  end
87
85
 
88
86
 
89
-
90
- def add_transaction_trace_info(request, response)
87
+ def add_transaction_trace_info(request, response, cross_app)
91
88
  state = TingYun::Agent::TransactionState.tl_get
92
- filtered_uri = TingYun::Agent::HTTPClients::URIUtil.filter_uri request.uri
93
- transaction_sampler.add_node_info(:uri => filtered_uri)
94
- if response && response_is_cross_app?( response )
95
- transaction_sampler.tl_builder.current_node[:txId] = state.client_transaction_id || state.request_guid
96
- my_data = TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
97
- transaction_sampler.tl_builder.current_node[:txData] = my_data
89
+ ::TingYun::Agent::Collector::TransactionSampler.add_node_info(:uri => TingYun::Agent::HTTPClients::URIUtil.filter_uri(request.uri))
90
+ if cross_app
91
+ ::TingYun::Agent::Collector::TransactionSampler.tl_builder.set_txId_and_txData(state.client_transaction_id || state.request_guid,
92
+ TingYun::Support::Serialize::JSONWrapper.load(response[TY_DATA_HEADER].gsub("'",'"')))
98
93
  end
99
94
  end
100
95
 
101
- def metrics_for(state, request, response)
102
- metrics = common_metrics(request)
103
96
 
104
- if response && response_is_cross_app?( response )
97
+ def metrics_for(request, response, cross_app)
98
+
99
+ metrics = [ "External/NULL/ALL" ]
100
+
101
+ if TingYun::Agent::Transaction.recording_web_transaction?
102
+ metrics << "External/NULL/AllWeb"
103
+ else
104
+ metrics << "External/NULL/AllBackground"
105
+ end
106
+
107
+ if cross_app
105
108
  begin
106
109
  metrics.concat metrics_for_cross_app_response( request, response )
107
110
  rescue => err
@@ -117,20 +120,8 @@ module TingYun
117
120
  return metrics
118
121
  end
119
122
 
120
- def common_metrics(request)
121
- metrics = [ "External/NULL/ALL" ]
122
-
123
- if TingYun::Agent::Transaction.recording_web_transaction?
124
- metrics << "External/NULL/AllWeb"
125
- else
126
- metrics << "External/NULL/AllBackground"
127
- end
128
-
129
- return metrics
130
- end
131
123
 
132
124
  def metrics_for_regular_request( request )
133
- state = TingYun::Agent::TransactionState.tl_get
134
125
  metrics = []
135
126
  metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
136
127
  metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
@@ -138,29 +129,11 @@ module TingYun
138
129
  return metrics
139
130
  end
140
131
 
141
- def stats_engine
142
- ::TingYun::Agent.instance.stats_engine
143
- end
144
-
145
- def transaction_sampler
146
- ::TingYun::Agent.instance.transaction_sampler
147
- end
148
-
149
132
 
150
133
  def cross_app_enabled?
151
- valid_tingyun_secret_id? && web_action_tracer_enabled? && cross_application_tracer_enabled?
152
- end
153
-
154
- def web_action_tracer_enabled?
155
- TingYun::Agent.config[:'nbs.action_tracer.enabled']
156
- end
157
-
158
- def cross_application_tracer_enabled?
159
- TingYun::Agent.config[:'nbs.transaction_tracer.enabled']
160
- end
161
-
162
- def valid_tingyun_secret_id?
163
- TingYun::Agent.config[:tingyunIdSecret] && TingYun::Agent.config[:tingyunIdSecret].size > 0
134
+ TingYun::Agent.config[:tingyunIdSecret] && TingYun::Agent.config[:tingyunIdSecret].size > 0 &&
135
+ TingYun::Agent.config[:'nbs.action_tracer.enabled'] &&
136
+ TingYun::Agent.config[:'nbs.transaction_tracer.enabled']
164
137
  end
165
138
 
166
139
  # Inject the X-Process header into the outgoing +request+.
@@ -168,25 +141,22 @@ module TingYun
168
141
  cross_app_id = TingYun::Agent.config[:tingyunIdSecret] or
169
142
  raise TingYun::Agent::CrossAppTracing::Error, "no tingyunIdSecret configured"
170
143
 
171
- txn_guid = state.client_transaction_id || state.request_guid
172
- state.transaction_sample_builder.trace.tx_id = txn_guid
173
- request[TY_ID_HEADER] = "#{cross_app_id};c=1;x=#{txn_guid}"
144
+ request[TY_ID_HEADER] = "#{cross_app_id};c=1;x=#{state.request_guid}"
174
145
  end
175
146
 
176
147
  # Returns +true+ if Cross Application Tracing is enabled, and the given +response+
177
148
  # has the appropriate headers.
178
149
  def response_is_cross_app?( response )
150
+ return false unless response
151
+ return false unless response[TY_DATA_HEADER]
179
152
  return false unless cross_app_enabled?
180
- unless response[TY_DATA_HEADER]
181
- return false
182
- end
153
+
183
154
  return true
184
155
  end
185
156
 
186
157
  # Return the set of metric objects appropriate for the given cross app
187
158
  # +response+.
188
159
  def metrics_for_cross_app_response(request, response )
189
- state = TingYun::Agent::TransactionState.tl_get
190
160
  my_data = TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
191
161
  uri = "#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
192
162
  metrics = []
@@ -1,8 +1,13 @@
1
1
  # encoding: utf-8
2
2
 
3
3
  require 'ting_yun/support/helper'
4
+ require 'ting_yun/agent/database/connection_manager'
5
+ require 'ting_yun/agent/database/statement'
6
+ require 'ting_yun/agent/database/obfuscator'
7
+
4
8
  module TingYun
5
9
  module Agent
10
+ # sql explain plan
6
11
  module Database
7
12
 
8
13
  MAX_QUERY_LENGTH = 16384
@@ -10,8 +15,31 @@ module TingYun
10
15
  extend self
11
16
 
12
17
 
18
+ def explain_sql(statement)
19
+ return nil unless statement.sql && statement.explainer && statement.config
20
+ statement.sql = statement.sql.split(";\n")[0] # only explain the first
21
+ return statement.explain || {"dialect"=> nil, "keys"=>[], "values"=>[]}
22
+ end
23
+
24
+ def explain_plan(statement)
25
+ connection = get_connection(statement.config) do
26
+ ::ActiveRecord::Base.send("#{statement.config[:adapter]}_connection",
27
+ statement.config)
28
+ end
29
+ if connection
30
+ if connection.respond_to?(:exec_query)
31
+ return connection.exec_query("EXPLAIN #{statement.sql}",
32
+ "Explain #{statement.name}",
33
+ statement.binds)
34
+ elsif connection.respond_to?(:execute)
35
+ return connection.execute("EXPLAIN #{statement.sql}")
36
+ end
37
+ end
38
+ end
39
+
40
+
13
41
  def obfuscate_sql(sql)
14
- Obfuscator.instance.obfuscator.call(sql)
42
+ TingYun::Agent::Database::Obfuscator.instance.obfuscator.call(sql)
15
43
  end
16
44
 
17
45
 
@@ -28,11 +56,6 @@ module TingYun
28
56
  end
29
57
 
30
58
 
31
- RECORD_FOR = [:raw, :obfuscated].freeze
32
-
33
- def should_record_sql?(key)
34
- RECORD_FOR.include?(record_sql_method(key.to_sym))
35
- end
36
59
 
37
60
  def record_sql_method(key)
38
61
 
@@ -46,363 +69,32 @@ module TingYun
46
69
  end
47
70
  end
48
71
 
49
- def should_action_collect_explain_plans?
50
- should_record_sql?("nbs.action_tracer.record_sql") &&
51
- Agent.config["nbs.action_tracer.explain_enabled".to_sym]
52
- end
53
-
54
- def explain_sql(sql, config, explainer=nil)
55
- return nil unless sql && explainer && config
56
- _sql = sql.split(";\n")[0] # only explain the first
57
- explain_plan = explain(_sql, config, explainer)
58
- return explain_plan || {"dialect"=> nil, "keys"=>[], "values"=>[]}
59
- end
60
-
61
- SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
62
-
63
- def explain(sql, config, explainer=nil)
64
-
65
- return unless explainer && is_select?(sql)
66
-
67
- if sql[-3,3] == '...'
68
- TingYun::Agent.logger.debug('Unable to collect explain plan for truncated query.')
69
- return
70
- end
71
-
72
- if parameterized?(sql)
73
- TingYun::Agent.logger.debug('Unable to collect explain plan for parameterized query.')
74
- return
75
- end
76
-
77
- adapter = adapter_from_config(config)
78
- if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
79
- TingYun::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
80
- return
81
- end
82
-
83
- handle_exception_in_explain do
84
- plan = explainer.call(config, sql)
85
- return process_resultset(plan, adapter) if plan
86
- end
87
- end
88
-
89
- def adapter_from_config(config)
90
- if config[:adapter]
91
- return config[:adapter].to_s
92
- elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
93
- # This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
94
- # jdbc-sqlite3 gems.
95
- return $1
96
- end
97
- end
98
-
99
-
100
- def parameterized?(sql)
101
- Obfuscator.instance.obfuscate_single_quote_literals(sql) =~ /\$\d+/
102
- end
103
-
104
- def is_select?(sql)
105
- parse_operation_from_query(sql) == 'select'
106
- end
107
-
108
- def process_resultset(results ,adapter)
109
- case adapter.to_s
110
- when 'postgres', 'postgresql'
111
- process_explain_results_postgres(results)
112
- when 'mysql2'
113
- process_explain_results_mysql2(results)
114
- when 'mysql'
115
- process_explain_results_mysql(results)
116
- when 'sqlite'
117
- process_explain_results_sqlite(results)
118
- end
119
- end
120
-
121
- QUERY_PLAN = 'QUERY PLAN'.freeze
122
-
123
- def process_explain_results_postgres(results)
124
- if results.is_a?(String)
125
- query_plan_string = results
126
- else
127
- lines = []
128
- results.each { |row| lines << row[QUERY_PLAN] }
129
- query_plan_string = lines.join("\n")
130
- end
131
-
132
- unless record_sql_method("nbs.action_tracer.record_sql") == :raw
133
- query_plan_string = Obfuscator.instance.obfuscate_postgres_explain(query_plan_string)
134
- end
135
- values = query_plan_string.split("\n").map { |line| [line] }
136
-
137
- {"dialect"=> "PostgreSQL", "keys"=>[QUERY_PLAN], "values"=>values}
138
- end
139
-
140
- def string_explain_plan_results(adpater, results)
141
- {"dialect"=> adpater, "keys"=>[], "values"=>[results]}
142
- end
143
-
144
- def process_explain_results_mysql2(results)
145
- return string_explain_plan_results("MySQL", results) if results.is_a?(String)
146
- headers = results.fields
147
- values = []
148
- results.each { |row| values << row }
149
- {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
150
- end
151
-
152
- def process_explain_results_mysql(results)
153
- return string_explain_plan_results("MySQL", results) if results.is_a?(String)
154
- headers = []
155
- values = []
156
- if results.is_a?(Array)
157
- # We're probably using the jdbc-mysql gem for JRuby, which will give
158
- # us an array of hashes.
159
- headers = results.first.keys
160
- results.each do |row|
161
- values << headers.map { |h| row[h] }
162
- end
163
- else
164
- # We're probably using the native mysql driver gem, which will give us
165
- # a Mysql::Result object that responds to each_hash
166
- results.each_hash do |row|
167
- headers = row.keys
168
- values << headers.map { |h| row[h] }
169
- end
170
- end
171
- {"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
172
- end
173
-
174
- SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
175
-
176
- def process_explain_results_sqlite(results)
177
- return string_explain_plan_results("sqlite", results) if results.is_a?(String)
178
- headers = SQLITE_EXPLAIN_COLUMNS
179
- values = []
180
- results.each do |row|
181
- values << headers.map { |h| row[h] }
182
- end
183
- {"dialect"=> "sqlite", "keys"=>headers, "values"=>values}
184
- end
185
-
186
-
187
-
188
- KNOWN_OPERATIONS = [
189
- 'alter',
190
- 'select',
191
- 'update',
192
- 'delete',
193
- 'insert',
194
- 'create',
195
- 'show',
196
- 'set',
197
- 'exec',
198
- 'execute',
199
- 'call'
200
- ]
201
-
202
- SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
203
- EMPTY_STRING = ''.freeze
204
-
205
- def parse_operation_from_query(sql)
206
- sql = TingYun::Helper.correctly_encoded(sql).gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
207
- if sql =~ /(\w+)/
208
- op = $1.downcase
209
- return op if KNOWN_OPERATIONS.include?(op)
210
- end
211
- end
212
-
213
-
214
- def handle_exception_in_explain
215
- yield
216
- rescue => e
217
- ::TingYun::Agent.logger.error("Error getting query plan:", e)
218
- nil
219
- end
220
-
221
72
 
222
73
  def get_connection(config, &connector)
223
- ConnectionManager.instance.get_connection(config, &connector)
74
+ TingYun::Agent::Database::ConnectionManager.instance.get_connection(config, &connector)
224
75
  end
225
76
 
226
77
  def close_connections
227
- ConnectionManager.instance.close_connections
78
+ TingYun::Agent::Database::ConnectionManager.instance.close_connections
228
79
  end
229
80
 
230
- # Returns a cached connection for a given ActiveRecord
231
- # configuration - these are stored or reopened as needed, and if
232
- # we cannot get one, we ignore it and move on without explaining
233
- # the sql
234
- class ConnectionManager
235
- include Singleton
236
-
237
- def get_connection(config, &connector)
238
- @connections ||= {}
239
-
240
- connection = @connections[config]
241
-
242
- return connection if connection
243
81
 
244
- begin
245
- @connections[config] = connector.call(config)
246
- rescue => e
247
- ::TingYun::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
248
- nil
249
- end
250
- end
251
82
 
252
- # Closes all the connections in the internal connection cache
253
- def close_connections
254
- @connections ||= {}
255
- @connections.values.each do |connection|
256
- begin
257
- connection.disconnect!
258
- rescue
259
- end
260
- end
83
+ RECORD_FOR = [:raw, :obfuscated].freeze
261
84
 
262
- @connections = {}
263
- end
85
+ def should_record_sql?(key)
86
+ RECORD_FOR.include?(record_sql_method(key.to_sym))
264
87
  end
265
88
 
266
- class Statement
267
- attr_accessor :sql, :config, :explainer
268
-
269
- def initialize(sql, config={}, explainer=nil)
270
- @sql = TingYun::Agent::Database.capture_query(sql)
271
- @config = config
272
- @explainer = explainer
273
- end
274
-
275
- def adapter
276
- config && config[:adapter]
277
- end
89
+ def sql_sampler_enabled?
90
+ Agent.config[:'nbs.action_tracer.enabled'] &&
91
+ Agent.config[:'nbs.action_tracer.slow_sql'] &&
92
+ should_record_sql?('nbs.action_tracer.record_sql')
278
93
  end
279
94
 
280
- #混淆器
281
- class Obfuscator
282
- include Singleton
283
-
284
- attr_reader :obfuscator
285
-
286
- def initialize
287
- reset
288
- end
289
-
290
- def reset
291
- @obfuscator = method(:default_sql_obfuscator)
292
- end
293
-
294
- QUERY_TOO_LARGE_MESSAGE = "Query too large (over 16k characters) to safely obfuscate"
295
- FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
296
-
297
- def default_sql_obfuscator(sql)
298
- stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
299
-
300
- if stmt.sql[-3,3] == '...'
301
- return QUERY_TOO_LARGE_MESSAGE
302
- end
303
-
304
- obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
305
-
306
- obfuscated = obfuscate_numeric_literals(stmt.sql)
307
-
308
- if obfuscate_double_quotes
309
- obfuscated = obfuscate_quoted_literals(obfuscated)
310
- obfuscated = remove_comments(obfuscated)
311
- if contains_quotes?(obfuscated)
312
- obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
313
- end
314
- else
315
- obfuscated = obfuscate_single_quote_literals(obfuscated)
316
- obfuscated = remove_comments(obfuscated)
317
- if contains_single_quotes?(obfuscated)
318
- obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
319
- end
320
- end
321
-
322
-
323
- obfuscated.to_s # return back to a regular String
324
- end
325
-
326
- QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
327
- LABEL_LINE_REGEX = /^([^:\n]*:\s+).*$/.freeze
328
-
329
- def obfuscate_postgres_explain(explain)
330
- explain.gsub!(QUOTED_STRINGS_REGEX) do |match|
331
- match.start_with?('"') ? match : '?'
332
- end
333
- explain.gsub!(LABEL_LINE_REGEX, '\1?')
334
- explain
335
- end
336
-
337
- module ObfuscationHelpers
338
- # Note that the following two regexes are applied to a reversed version
339
- # of the query. This is why the backslash escape sequences (\' and \")
340
- # appear reversed within them.
341
- #
342
- # Note that some database adapters (notably, PostgreSQL with
343
- # standard_conforming_strings on and MySQL with NO_BACKSLASH_ESCAPES on)
344
- # do not apply special treatment to backslashes within quoted string
345
- # literals. We don't have an easy way of determining whether the
346
- # database connection from which a query was captured was operating in
347
- # one of these modes, but the obfuscation is done in such a way that it
348
- # should not matter.
349
- #
350
- # Reversing the query string before obfuscation allows us to get around
351
- # the fact that a \' appearing within a string may or may not terminate
352
- # the string, because we know that a string cannot *start* with a \'.
353
- REVERSE_SINGLE_QUOTES_REGEX = /'(?:''|'\\|[^'])*'/
354
- REVERSE_ANY_QUOTES_REGEX = /'(?:''|'\\|[^'])*'|"(?:""|"\\|[^"])*"/
355
-
356
- NUMERICS_REGEX = /\b\d+\b/
357
-
358
- # We take a conservative, overly-aggressive approach to obfuscating
359
- # comments, and drop everything from the query after encountering any
360
- # character sequence that could be a comment initiator. We do this after
361
- # removal of string literals to avoid accidentally over-obfuscating when
362
- # a string literal contains a comment initiator.
363
- SQL_COMMENT_REGEX = Regexp.new('(?:/\*|--|#).*', Regexp::MULTILINE).freeze
364
-
365
- # We use these to check whether the query contains any quote characters
366
- # after obfuscation. If so, that's a good indication that the original
367
- # query was malformed, and so our obfuscation can't reliabily find
368
- # literals. In such a case, we'll replace the entire query with a
369
- # placeholder.
370
- LITERAL_SINGLE_QUOTE = "'".freeze
371
- LITERAL_DOUBLE_QUOTE = '"'.freeze
372
-
373
- PLACEHOLDER = '?'.freeze
374
-
375
- def obfuscate_single_quote_literals(sql)
376
- obfuscated = sql.reverse
377
- obfuscated.gsub!(REVERSE_SINGLE_QUOTES_REGEX, PLACEHOLDER)
378
- obfuscated.reverse!
379
- obfuscated
380
- end
381
-
382
- def obfuscate_quoted_literals(sql)
383
- obfuscated = sql.reverse
384
- obfuscated.gsub!(REVERSE_ANY_QUOTES_REGEX, PLACEHOLDER)
385
- obfuscated.reverse!
386
- obfuscated
387
- end
388
-
389
- def obfuscate_numeric_literals(sql)
390
- sql.gsub(NUMERICS_REGEX, PLACEHOLDER)
391
- end
392
-
393
- def remove_comments(sql)
394
- sql.gsub(SQL_COMMENT_REGEX, PLACEHOLDER)
395
- end
396
-
397
- def contains_single_quotes?(str)
398
- str.include?(LITERAL_SINGLE_QUOTE)
399
- end
400
-
401
- def contains_quotes?(str)
402
- str.include?(LITERAL_SINGLE_QUOTE) || str.include?(LITERAL_DOUBLE_QUOTE)
403
- end
404
- end
405
- include ObfuscationHelpers
95
+ def should_action_collect_explain_plans?
96
+ should_record_sql?("nbs.action_tracer.record_sql") &&
97
+ Agent.config["nbs.action_tracer.explain_enabled".to_sym]
406
98
  end
407
99
 
408
100
  end