tingyun_rpm 1.1.4.2 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -1
- data/Guardfile +10 -0
- data/lib/ting_yun/agent.rb +1 -0
- data/lib/ting_yun/agent/agent.rb +16 -27
- data/lib/ting_yun/agent/collector/error_collector.rb +7 -18
- data/lib/ting_yun/agent/collector/error_collector/noticed_error.rb +26 -21
- data/lib/ting_yun/agent/collector/middle_ware_collector/cpu_sampler.rb +4 -9
- data/lib/ting_yun/agent/collector/sql_sampler.rb +32 -188
- data/lib/ting_yun/agent/collector/sql_sampler/slow_sql.rb +47 -0
- data/lib/ting_yun/agent/collector/sql_sampler/sql_trace.rb +73 -0
- data/lib/ting_yun/agent/collector/sql_sampler/transaction_sql_data.rb +26 -0
- data/lib/ting_yun/agent/collector/stats_engine/metric_stats.rb +6 -5
- data/lib/ting_yun/agent/collector/stats_engine/stats_hash.rb +2 -2
- data/lib/ting_yun/agent/collector/transaction_sampler.rb +23 -159
- data/lib/ting_yun/agent/collector/transaction_sampler/class_method.rb +130 -0
- data/lib/ting_yun/agent/collector/transaction_sampler/slowest_sample_buffer.rb +1 -1
- data/lib/ting_yun/agent/collector/transaction_sampler/transaction_sample_buffer_base.rb +1 -1
- data/lib/ting_yun/agent/cross_app/cross_app_monitor.rb +29 -79
- data/lib/ting_yun/agent/cross_app/cross_app_tracing.rb +36 -66
- data/lib/ting_yun/agent/database.rb +41 -349
- data/lib/ting_yun/agent/database/connection_manager.rb +44 -0
- data/lib/ting_yun/agent/database/explain_plan_helpers.rb +173 -0
- data/lib/ting_yun/agent/database/obfuscator.rb +151 -0
- data/lib/ting_yun/agent/database/statement.rb +70 -0
- data/lib/ting_yun/agent/event/event_loop.rb +1 -2
- data/lib/ting_yun/agent/instance_methods/connect.rb +8 -20
- data/lib/ting_yun/agent/instance_methods/container_data_manager.rb +2 -3
- data/lib/ting_yun/agent/instance_methods/handle_errors.rb +6 -1
- data/lib/ting_yun/agent/instance_methods/start.rb +13 -81
- data/lib/ting_yun/agent/transaction.rb +48 -391
- data/lib/ting_yun/agent/transaction/apdex.rb +53 -0
- data/lib/ting_yun/agent/transaction/attributes.rb +2 -1
- data/lib/ting_yun/agent/transaction/class_method.rb +127 -0
- data/lib/ting_yun/agent/transaction/exceptions.rb +42 -0
- data/lib/ting_yun/agent/transaction/instance_method.rb +139 -0
- data/lib/ting_yun/agent/transaction/request_attributes.rb +9 -39
- data/lib/ting_yun/agent/transaction/trace.rb +7 -5
- data/lib/ting_yun/agent/transaction/trace_node.rb +1 -3
- data/lib/ting_yun/agent/transaction/traced_method_stack.rb +2 -3
- data/lib/ting_yun/agent/transaction/transaction_sample_builder.rb +6 -1
- data/lib/ting_yun/agent/transaction/transaction_state.rb +59 -17
- data/lib/ting_yun/agent/transaction/transaction_timings.rb +72 -0
- data/lib/ting_yun/configuration.rb +11 -0
- data/lib/ting_yun/configuration/default_source.rb +20 -17
- data/lib/ting_yun/configuration/manager.rb +50 -21
- data/lib/ting_yun/frameworks.rb +1 -0
- data/lib/ting_yun/frameworks/rails.rb +15 -0
- data/lib/ting_yun/instrumentation/active_record.rb +12 -18
- data/lib/ting_yun/instrumentation/middleware_tracing.rb +8 -14
- data/lib/ting_yun/instrumentation/mongo.rb +21 -27
- data/lib/ting_yun/instrumentation/mongo_command_log_subscriber.rb +7 -3
- data/lib/ting_yun/instrumentation/moped.rb +2 -2
- data/lib/ting_yun/instrumentation/net.rb +4 -5
- data/lib/ting_yun/instrumentation/rack.rb +1 -2
- data/lib/ting_yun/instrumentation/rails4/active_record_subscriber.rb +22 -20
- data/lib/ting_yun/instrumentation/redis.rb +2 -2
- data/lib/ting_yun/instrumentation/support/controller_instrumentation.rb +1 -1
- data/lib/ting_yun/instrumentation/support/external_error.rb +19 -16
- data/lib/ting_yun/instrumentation/support/javascript_instrumentor.rb +92 -0
- data/lib/ting_yun/instrumentation/support/thrift_helper.rb +73 -0
- data/lib/ting_yun/instrumentation/thrift.rb +19 -222
- data/lib/ting_yun/logger.rb +1 -0
- data/lib/ting_yun/logger/agent_logger.rb +11 -67
- data/lib/ting_yun/logger/create_logger_helper.rb +72 -0
- data/lib/ting_yun/metrics/metric_data.rb +9 -31
- data/lib/ting_yun/metrics/metric_spec.rb +11 -0
- data/lib/ting_yun/metrics/stats.rb +24 -1
- data/lib/ting_yun/middleware/agent_middleware.rb +28 -0
- data/lib/ting_yun/middleware/browser_monitoring.rb +111 -0
- data/lib/ting_yun/support/coerce.rb +1 -0
- data/lib/ting_yun/support/exception.rb +2 -33
- data/lib/ting_yun/support/local_environment.rb +7 -7
- data/lib/ting_yun/support/serialize/marshaller.rb +7 -25
- data/lib/ting_yun/ting_yun_service.rb +12 -9
- data/lib/ting_yun/ting_yun_service/connection.rb +3 -0
- data/lib/ting_yun/ting_yun_service/http.rb +4 -1
- data/lib/ting_yun/ting_yun_service/request.rb +5 -13
- data/lib/ting_yun/ting_yun_service/upload_service.rb +5 -7
- data/lib/ting_yun/version.rb +3 -5
- data/lib/tingyun_rpm.rb +12 -10
- data/tingyun_rpm.gemspec +3 -0
- metadata +49 -5
- data/.DS_Store +0 -0
- data/lib/ting_yun/agent/collector/base_sampler.rb +0 -2
@@ -19,113 +19,63 @@ module TingYun
|
|
19
19
|
register_event_listeners(events)
|
20
20
|
end
|
21
21
|
|
22
|
+
|
22
23
|
# Expected sequence of events:
|
23
24
|
# :before_call will save our cross application request id to the thread
|
24
25
|
# :after_call will write our response headers/metrics and clean up the thread
|
25
26
|
def register_event_listeners(events)
|
26
27
|
TingYun::Agent.logger.debug("Wiring up Cross Application Tracing to events after finished configuring")
|
27
28
|
|
28
|
-
events.subscribe(:
|
29
|
-
if cross_app_enabled?
|
29
|
+
events.subscribe(:cross_app_before_call) do |env| #THREAD_LOCAL_ACCESS
|
30
|
+
if TingYun::Agent::CrossAppTracing.cross_app_enabled?
|
30
31
|
state = TingYun::Agent::TransactionState.tl_get
|
31
|
-
save_referring_transaction_info(
|
32
|
+
state.save_referring_transaction_info(env[TY_ID_HEADER].split(';')) if env[TY_ID_HEADER]
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
35
|
-
events.subscribe(:
|
36
|
-
|
37
|
-
state.queue_duration = state.current_transaction.queue_time * 1000
|
38
|
-
state.web_duration = (Time.now - state.current_transaction.start_time) * 1000
|
39
|
-
insert_response_header(state, headers)
|
36
|
+
events.subscribe(:cross_app_after_call) do |_status_code, headers, _body| #THREAD_LOCAL_ACCESS
|
37
|
+
insert_response_header(headers) if TingYun::Agent::CrossAppTracing.cross_app_enabled?
|
40
38
|
end
|
41
39
|
|
42
40
|
end
|
43
41
|
|
44
42
|
|
45
|
-
def
|
46
|
-
TingYun::Agent::
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
def save_referring_transaction_info(state,request)
|
51
|
-
|
52
|
-
info = request[TY_ID_HEADER].split(';')
|
53
|
-
tingyun_id_secret = info[0]
|
54
|
-
client_transaction_id = info.find do |e|
|
55
|
-
e.match(/x=/)
|
56
|
-
end.split('=')[1] rescue nil
|
57
|
-
client_req_id = info.find do |e|
|
58
|
-
e.match(/r=/)
|
59
|
-
end.split('=')[1] rescue nil
|
60
|
-
|
61
|
-
state.client_tingyun_id_secret = tingyun_id_secret
|
62
|
-
state.client_transaction_id = client_transaction_id
|
63
|
-
state.transaction_sample_builder.trace.tx_id = client_transaction_id
|
64
|
-
state.client_req_id = client_req_id
|
65
|
-
end
|
66
|
-
|
67
|
-
|
68
|
-
def insert_response_header(state, response_headers)
|
69
|
-
if same_account?(state)
|
43
|
+
def insert_response_header(response_headers)
|
44
|
+
state = TingYun::Agent::TransactionState.tl_get
|
45
|
+
if state.same_account?
|
70
46
|
txn = state.current_transaction
|
71
|
-
|
72
|
-
set_response_headers
|
47
|
+
if txn
|
48
|
+
# set_response_headers
|
49
|
+
response_headers[TY_DATA_HEADER] = TingYun::Support::Serialize::JSONWrapper.dump build_payload(state)
|
50
|
+
TingYun::Agent.logger.debug("now,cross app will send response_headers #{response_headers[TY_DATA_HEADER]}")
|
73
51
|
end
|
74
|
-
clear_client_tingyun_id_secret(state)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
def clear_client_tingyun_id_secret(state)
|
79
|
-
state.client_tingyun_id_secret = nil
|
80
|
-
end
|
81
|
-
|
82
|
-
def same_account?(state)
|
83
|
-
server_info = TingYun::Agent.config[:tingyunIdSecret].split('|')
|
84
|
-
client_info = (state.client_tingyun_id_secret || '').split('|')
|
85
|
-
if !server_info[0].nil? && server_info[0] == client_info[0] && !server_info[0].empty?
|
86
|
-
return true
|
87
|
-
else
|
88
|
-
return false
|
89
52
|
end
|
90
53
|
end
|
91
54
|
|
92
|
-
def set_response_headers(state, response_headers)
|
93
|
-
response_headers[TY_DATA_HEADER] = TingYun::Support::Serialize::JSONWrapper.dump build_payload(state)
|
94
|
-
TingYun::Agent.logger.debug("now,cross app will send response_headers #{response_headers[TY_DATA_HEADER]}")
|
95
|
-
end
|
96
55
|
|
97
56
|
def build_payload(state)
|
57
|
+
timings = state.timings
|
58
|
+
|
98
59
|
payload = {
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
60
|
+
:id => TingYun::Agent.config[:tingyunIdSecret].split('|')[1],
|
61
|
+
:action => state.transaction_name,
|
62
|
+
:trId => state.trace_id,
|
63
|
+
:time => {
|
64
|
+
:duration => timings.app_time_in_millis,
|
65
|
+
:qu => timings.queue_time_in_millis,
|
66
|
+
:db => timings.sql_duration,
|
67
|
+
:ex => timings.external_duration,
|
68
|
+
:rds => timings.rds_duration,
|
69
|
+
:mc => timings.mc_duration,
|
70
|
+
:mon => timings.mon_duration,
|
71
|
+
:code => timings.app_execute_duration
|
72
|
+
}
|
112
73
|
}
|
113
|
-
payload[:tr] = 1 if slow_action_tracer?
|
74
|
+
payload[:tr] = 1 if timings.slow_action_tracer?
|
114
75
|
payload[:r] = state.client_req_id unless state.client_req_id.nil?
|
115
76
|
payload
|
116
77
|
end
|
117
78
|
|
118
|
-
def slow_action_tracer?(state)
|
119
|
-
if state.web_duration > TingYun::Agent.config[:'nbs.action_tracer.action_threshold']
|
120
|
-
return true
|
121
|
-
else
|
122
|
-
return false
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
def execute_duration(state)
|
127
|
-
state.web_duration - state.queue_duration - state.sql_duration - state.external_duration - state.rds_duration - state.mc_duration - state.mon_duration
|
128
|
-
end
|
129
79
|
end
|
130
80
|
end
|
131
81
|
end
|
@@ -5,6 +5,7 @@ require 'ting_yun/agent/transaction'
|
|
5
5
|
require 'ting_yun/support/http_clients/uri_util'
|
6
6
|
require 'ting_yun/support/serialize/json_wrapper'
|
7
7
|
require 'ting_yun/instrumentation/support/external_error'
|
8
|
+
require 'ting_yun/agent/collector/transaction_sampler'
|
8
9
|
|
9
10
|
|
10
11
|
module TingYun
|
@@ -22,26 +23,22 @@ module TingYun
|
|
22
23
|
TY_DATA_HEADER = 'X-Tingyun-Tx-Data'.freeze
|
23
24
|
|
24
25
|
|
25
|
-
|
26
|
-
|
27
26
|
module_function
|
28
27
|
|
29
28
|
|
30
|
-
|
31
29
|
def tl_trace_http_request(request)
|
32
|
-
t0 = Time.now.to_f
|
33
30
|
state = TingYun::Agent::TransactionState.tl_get
|
34
31
|
return yield unless state.execution_traced?
|
35
32
|
return yield unless state.current_transaction #如果还没有创建Transaction,就发生跨应用,就直接先跳过跟踪。
|
36
33
|
|
34
|
+
t0 = Time.now.to_f
|
37
35
|
begin
|
38
36
|
node = start_trace(state, t0, request)
|
39
37
|
response = yield
|
40
|
-
capture_exception(response,request,'net%2Fhttp')
|
38
|
+
capture_exception(response, request, 'net%2Fhttp')
|
41
39
|
rescue => e
|
42
40
|
klass = "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
|
43
|
-
handle_error(e,klass)
|
44
|
-
raise e
|
41
|
+
handle_error(e, klass)
|
45
42
|
ensure
|
46
43
|
finish_trace(state, t0, node, request, response)
|
47
44
|
end
|
@@ -51,7 +48,7 @@ module TingYun
|
|
51
48
|
def start_trace(state, t0, request)
|
52
49
|
inject_request_headers(state, request) if cross_app_enabled?
|
53
50
|
stack = state.traced_method_stack
|
54
|
-
node = stack.push_frame(state
|
51
|
+
node = stack.push_frame(state, :http_request, t0)
|
55
52
|
|
56
53
|
return node
|
57
54
|
end
|
@@ -59,49 +56,55 @@ module TingYun
|
|
59
56
|
def finish_trace(state, t0, node, request, response)
|
60
57
|
t1 = Time.now.to_f
|
61
58
|
duration = (t1- t0) * 1000
|
62
|
-
state.external_duration = duration
|
59
|
+
state.timings.external_duration = duration
|
63
60
|
|
64
61
|
begin
|
65
62
|
if request
|
66
|
-
|
63
|
+
cross_app = response_is_cross_app?(response)
|
64
|
+
|
65
|
+
metrics = metrics_for(request, response, cross_app)
|
67
66
|
node_name = metrics.pop
|
68
67
|
scoped_metric = metrics.pop
|
69
68
|
|
70
|
-
stats_engine.record_scoped_and_unscoped_metrics(state, scoped_metric, metrics, duration)
|
69
|
+
::TingYun::Agent.instance.stats_engine.record_scoped_and_unscoped_metrics(state, scoped_metric, metrics, duration)
|
71
70
|
|
72
71
|
if node
|
73
72
|
node.name = node_name
|
74
|
-
add_transaction_trace_info(request, response)
|
73
|
+
add_transaction_trace_info(request, response, cross_app)
|
75
74
|
end
|
76
75
|
end
|
76
|
+
rescue => err
|
77
|
+
TingYun::Agent.logger.error "Uncaught exception while finishing an HTTP request trace", err
|
77
78
|
ensure
|
78
79
|
if node
|
79
80
|
stack = state.traced_method_stack
|
80
81
|
stack.pop_frame(state, node, node_name, t1)
|
81
82
|
end
|
82
83
|
end
|
83
|
-
rescue => err
|
84
|
-
TingYun::Agent.logger.error "Uncaught exception while finishing an HTTP request trace", err
|
85
|
-
raise
|
86
84
|
end
|
87
85
|
|
88
86
|
|
89
|
-
|
90
|
-
def add_transaction_trace_info(request, response)
|
87
|
+
def add_transaction_trace_info(request, response, cross_app)
|
91
88
|
state = TingYun::Agent::TransactionState.tl_get
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
my_data = TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
|
97
|
-
transaction_sampler.tl_builder.current_node[:txData] = my_data
|
89
|
+
::TingYun::Agent::Collector::TransactionSampler.add_node_info(:uri => TingYun::Agent::HTTPClients::URIUtil.filter_uri(request.uri))
|
90
|
+
if cross_app
|
91
|
+
::TingYun::Agent::Collector::TransactionSampler.tl_builder.set_txId_and_txData(state.client_transaction_id || state.request_guid,
|
92
|
+
TingYun::Support::Serialize::JSONWrapper.load(response[TY_DATA_HEADER].gsub("'",'"')))
|
98
93
|
end
|
99
94
|
end
|
100
95
|
|
101
|
-
def metrics_for(state, request, response)
|
102
|
-
metrics = common_metrics(request)
|
103
96
|
|
104
|
-
|
97
|
+
def metrics_for(request, response, cross_app)
|
98
|
+
|
99
|
+
metrics = [ "External/NULL/ALL" ]
|
100
|
+
|
101
|
+
if TingYun::Agent::Transaction.recording_web_transaction?
|
102
|
+
metrics << "External/NULL/AllWeb"
|
103
|
+
else
|
104
|
+
metrics << "External/NULL/AllBackground"
|
105
|
+
end
|
106
|
+
|
107
|
+
if cross_app
|
105
108
|
begin
|
106
109
|
metrics.concat metrics_for_cross_app_response( request, response )
|
107
110
|
rescue => err
|
@@ -117,20 +120,8 @@ module TingYun
|
|
117
120
|
return metrics
|
118
121
|
end
|
119
122
|
|
120
|
-
def common_metrics(request)
|
121
|
-
metrics = [ "External/NULL/ALL" ]
|
122
|
-
|
123
|
-
if TingYun::Agent::Transaction.recording_web_transaction?
|
124
|
-
metrics << "External/NULL/AllWeb"
|
125
|
-
else
|
126
|
-
metrics << "External/NULL/AllBackground"
|
127
|
-
end
|
128
|
-
|
129
|
-
return metrics
|
130
|
-
end
|
131
123
|
|
132
124
|
def metrics_for_regular_request( request )
|
133
|
-
state = TingYun::Agent::TransactionState.tl_get
|
134
125
|
metrics = []
|
135
126
|
metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
|
136
127
|
metrics << "External/#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
|
@@ -138,29 +129,11 @@ module TingYun
|
|
138
129
|
return metrics
|
139
130
|
end
|
140
131
|
|
141
|
-
def stats_engine
|
142
|
-
::TingYun::Agent.instance.stats_engine
|
143
|
-
end
|
144
|
-
|
145
|
-
def transaction_sampler
|
146
|
-
::TingYun::Agent.instance.transaction_sampler
|
147
|
-
end
|
148
|
-
|
149
132
|
|
150
133
|
def cross_app_enabled?
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
def web_action_tracer_enabled?
|
155
|
-
TingYun::Agent.config[:'nbs.action_tracer.enabled']
|
156
|
-
end
|
157
|
-
|
158
|
-
def cross_application_tracer_enabled?
|
159
|
-
TingYun::Agent.config[:'nbs.transaction_tracer.enabled']
|
160
|
-
end
|
161
|
-
|
162
|
-
def valid_tingyun_secret_id?
|
163
|
-
TingYun::Agent.config[:tingyunIdSecret] && TingYun::Agent.config[:tingyunIdSecret].size > 0
|
134
|
+
TingYun::Agent.config[:tingyunIdSecret] && TingYun::Agent.config[:tingyunIdSecret].size > 0 &&
|
135
|
+
TingYun::Agent.config[:'nbs.action_tracer.enabled'] &&
|
136
|
+
TingYun::Agent.config[:'nbs.transaction_tracer.enabled']
|
164
137
|
end
|
165
138
|
|
166
139
|
# Inject the X-Process header into the outgoing +request+.
|
@@ -168,25 +141,22 @@ module TingYun
|
|
168
141
|
cross_app_id = TingYun::Agent.config[:tingyunIdSecret] or
|
169
142
|
raise TingYun::Agent::CrossAppTracing::Error, "no tingyunIdSecret configured"
|
170
143
|
|
171
|
-
|
172
|
-
state.transaction_sample_builder.trace.tx_id = txn_guid
|
173
|
-
request[TY_ID_HEADER] = "#{cross_app_id};c=1;x=#{txn_guid}"
|
144
|
+
request[TY_ID_HEADER] = "#{cross_app_id};c=1;x=#{state.request_guid}"
|
174
145
|
end
|
175
146
|
|
176
147
|
# Returns +true+ if Cross Application Tracing is enabled, and the given +response+
|
177
148
|
# has the appropriate headers.
|
178
149
|
def response_is_cross_app?( response )
|
150
|
+
return false unless response
|
151
|
+
return false unless response[TY_DATA_HEADER]
|
179
152
|
return false unless cross_app_enabled?
|
180
|
-
|
181
|
-
return false
|
182
|
-
end
|
153
|
+
|
183
154
|
return true
|
184
155
|
end
|
185
156
|
|
186
157
|
# Return the set of metric objects appropriate for the given cross app
|
187
158
|
# +response+.
|
188
159
|
def metrics_for_cross_app_response(request, response )
|
189
|
-
state = TingYun::Agent::TransactionState.tl_get
|
190
160
|
my_data = TingYun::Support::Serialize::JSONWrapper.load response[TY_DATA_HEADER].gsub("'",'"')
|
191
161
|
uri = "#{request.uri.to_s.gsub('/','%2F')}/net%2Fhttp"
|
192
162
|
metrics = []
|
@@ -1,8 +1,13 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
|
3
3
|
require 'ting_yun/support/helper'
|
4
|
+
require 'ting_yun/agent/database/connection_manager'
|
5
|
+
require 'ting_yun/agent/database/statement'
|
6
|
+
require 'ting_yun/agent/database/obfuscator'
|
7
|
+
|
4
8
|
module TingYun
|
5
9
|
module Agent
|
10
|
+
# sql explain plan
|
6
11
|
module Database
|
7
12
|
|
8
13
|
MAX_QUERY_LENGTH = 16384
|
@@ -10,8 +15,31 @@ module TingYun
|
|
10
15
|
extend self
|
11
16
|
|
12
17
|
|
18
|
+
def explain_sql(statement)
|
19
|
+
return nil unless statement.sql && statement.explainer && statement.config
|
20
|
+
statement.sql = statement.sql.split(";\n")[0] # only explain the first
|
21
|
+
return statement.explain || {"dialect"=> nil, "keys"=>[], "values"=>[]}
|
22
|
+
end
|
23
|
+
|
24
|
+
def explain_plan(statement)
|
25
|
+
connection = get_connection(statement.config) do
|
26
|
+
::ActiveRecord::Base.send("#{statement.config[:adapter]}_connection",
|
27
|
+
statement.config)
|
28
|
+
end
|
29
|
+
if connection
|
30
|
+
if connection.respond_to?(:exec_query)
|
31
|
+
return connection.exec_query("EXPLAIN #{statement.sql}",
|
32
|
+
"Explain #{statement.name}",
|
33
|
+
statement.binds)
|
34
|
+
elsif connection.respond_to?(:execute)
|
35
|
+
return connection.execute("EXPLAIN #{statement.sql}")
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
|
13
41
|
def obfuscate_sql(sql)
|
14
|
-
Obfuscator.instance.obfuscator.call(sql)
|
42
|
+
TingYun::Agent::Database::Obfuscator.instance.obfuscator.call(sql)
|
15
43
|
end
|
16
44
|
|
17
45
|
|
@@ -28,11 +56,6 @@ module TingYun
|
|
28
56
|
end
|
29
57
|
|
30
58
|
|
31
|
-
RECORD_FOR = [:raw, :obfuscated].freeze
|
32
|
-
|
33
|
-
def should_record_sql?(key)
|
34
|
-
RECORD_FOR.include?(record_sql_method(key.to_sym))
|
35
|
-
end
|
36
59
|
|
37
60
|
def record_sql_method(key)
|
38
61
|
|
@@ -46,363 +69,32 @@ module TingYun
|
|
46
69
|
end
|
47
70
|
end
|
48
71
|
|
49
|
-
def should_action_collect_explain_plans?
|
50
|
-
should_record_sql?("nbs.action_tracer.record_sql") &&
|
51
|
-
Agent.config["nbs.action_tracer.explain_enabled".to_sym]
|
52
|
-
end
|
53
|
-
|
54
|
-
def explain_sql(sql, config, explainer=nil)
|
55
|
-
return nil unless sql && explainer && config
|
56
|
-
_sql = sql.split(";\n")[0] # only explain the first
|
57
|
-
explain_plan = explain(_sql, config, explainer)
|
58
|
-
return explain_plan || {"dialect"=> nil, "keys"=>[], "values"=>[]}
|
59
|
-
end
|
60
|
-
|
61
|
-
SUPPORTED_ADAPTERS_FOR_EXPLAIN = %w[postgres postgresql mysql2 mysql sqlite].freeze
|
62
|
-
|
63
|
-
def explain(sql, config, explainer=nil)
|
64
|
-
|
65
|
-
return unless explainer && is_select?(sql)
|
66
|
-
|
67
|
-
if sql[-3,3] == '...'
|
68
|
-
TingYun::Agent.logger.debug('Unable to collect explain plan for truncated query.')
|
69
|
-
return
|
70
|
-
end
|
71
|
-
|
72
|
-
if parameterized?(sql)
|
73
|
-
TingYun::Agent.logger.debug('Unable to collect explain plan for parameterized query.')
|
74
|
-
return
|
75
|
-
end
|
76
|
-
|
77
|
-
adapter = adapter_from_config(config)
|
78
|
-
if !SUPPORTED_ADAPTERS_FOR_EXPLAIN.include?(adapter)
|
79
|
-
TingYun::Agent.logger.debug("Not collecting explain plan because an unknown connection adapter ('#{adapter}') was used.")
|
80
|
-
return
|
81
|
-
end
|
82
|
-
|
83
|
-
handle_exception_in_explain do
|
84
|
-
plan = explainer.call(config, sql)
|
85
|
-
return process_resultset(plan, adapter) if plan
|
86
|
-
end
|
87
|
-
end
|
88
|
-
|
89
|
-
def adapter_from_config(config)
|
90
|
-
if config[:adapter]
|
91
|
-
return config[:adapter].to_s
|
92
|
-
elsif config[:uri] && config[:uri].to_s =~ /^jdbc:([^:]+):/
|
93
|
-
# This case is for Sequel with the jdbc-mysql, jdbc-postgres, or
|
94
|
-
# jdbc-sqlite3 gems.
|
95
|
-
return $1
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
|
100
|
-
def parameterized?(sql)
|
101
|
-
Obfuscator.instance.obfuscate_single_quote_literals(sql) =~ /\$\d+/
|
102
|
-
end
|
103
|
-
|
104
|
-
def is_select?(sql)
|
105
|
-
parse_operation_from_query(sql) == 'select'
|
106
|
-
end
|
107
|
-
|
108
|
-
def process_resultset(results ,adapter)
|
109
|
-
case adapter.to_s
|
110
|
-
when 'postgres', 'postgresql'
|
111
|
-
process_explain_results_postgres(results)
|
112
|
-
when 'mysql2'
|
113
|
-
process_explain_results_mysql2(results)
|
114
|
-
when 'mysql'
|
115
|
-
process_explain_results_mysql(results)
|
116
|
-
when 'sqlite'
|
117
|
-
process_explain_results_sqlite(results)
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
QUERY_PLAN = 'QUERY PLAN'.freeze
|
122
|
-
|
123
|
-
def process_explain_results_postgres(results)
|
124
|
-
if results.is_a?(String)
|
125
|
-
query_plan_string = results
|
126
|
-
else
|
127
|
-
lines = []
|
128
|
-
results.each { |row| lines << row[QUERY_PLAN] }
|
129
|
-
query_plan_string = lines.join("\n")
|
130
|
-
end
|
131
|
-
|
132
|
-
unless record_sql_method("nbs.action_tracer.record_sql") == :raw
|
133
|
-
query_plan_string = Obfuscator.instance.obfuscate_postgres_explain(query_plan_string)
|
134
|
-
end
|
135
|
-
values = query_plan_string.split("\n").map { |line| [line] }
|
136
|
-
|
137
|
-
{"dialect"=> "PostgreSQL", "keys"=>[QUERY_PLAN], "values"=>values}
|
138
|
-
end
|
139
|
-
|
140
|
-
def string_explain_plan_results(adpater, results)
|
141
|
-
{"dialect"=> adpater, "keys"=>[], "values"=>[results]}
|
142
|
-
end
|
143
|
-
|
144
|
-
def process_explain_results_mysql2(results)
|
145
|
-
return string_explain_plan_results("MySQL", results) if results.is_a?(String)
|
146
|
-
headers = results.fields
|
147
|
-
values = []
|
148
|
-
results.each { |row| values << row }
|
149
|
-
{"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
|
150
|
-
end
|
151
|
-
|
152
|
-
def process_explain_results_mysql(results)
|
153
|
-
return string_explain_plan_results("MySQL", results) if results.is_a?(String)
|
154
|
-
headers = []
|
155
|
-
values = []
|
156
|
-
if results.is_a?(Array)
|
157
|
-
# We're probably using the jdbc-mysql gem for JRuby, which will give
|
158
|
-
# us an array of hashes.
|
159
|
-
headers = results.first.keys
|
160
|
-
results.each do |row|
|
161
|
-
values << headers.map { |h| row[h] }
|
162
|
-
end
|
163
|
-
else
|
164
|
-
# We're probably using the native mysql driver gem, which will give us
|
165
|
-
# a Mysql::Result object that responds to each_hash
|
166
|
-
results.each_hash do |row|
|
167
|
-
headers = row.keys
|
168
|
-
values << headers.map { |h| row[h] }
|
169
|
-
end
|
170
|
-
end
|
171
|
-
{"dialect"=> "MySQL", "keys"=>headers, "values"=>values}
|
172
|
-
end
|
173
|
-
|
174
|
-
SQLITE_EXPLAIN_COLUMNS = %w[addr opcode p1 p2 p3 p4 p5 comment]
|
175
|
-
|
176
|
-
def process_explain_results_sqlite(results)
|
177
|
-
return string_explain_plan_results("sqlite", results) if results.is_a?(String)
|
178
|
-
headers = SQLITE_EXPLAIN_COLUMNS
|
179
|
-
values = []
|
180
|
-
results.each do |row|
|
181
|
-
values << headers.map { |h| row[h] }
|
182
|
-
end
|
183
|
-
{"dialect"=> "sqlite", "keys"=>headers, "values"=>values}
|
184
|
-
end
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
KNOWN_OPERATIONS = [
|
189
|
-
'alter',
|
190
|
-
'select',
|
191
|
-
'update',
|
192
|
-
'delete',
|
193
|
-
'insert',
|
194
|
-
'create',
|
195
|
-
'show',
|
196
|
-
'set',
|
197
|
-
'exec',
|
198
|
-
'execute',
|
199
|
-
'call'
|
200
|
-
]
|
201
|
-
|
202
|
-
SQL_COMMENT_REGEX = Regexp.new('/\*.*?\*/', Regexp::MULTILINE).freeze
|
203
|
-
EMPTY_STRING = ''.freeze
|
204
|
-
|
205
|
-
def parse_operation_from_query(sql)
|
206
|
-
sql = TingYun::Helper.correctly_encoded(sql).gsub(SQL_COMMENT_REGEX, EMPTY_STRING)
|
207
|
-
if sql =~ /(\w+)/
|
208
|
-
op = $1.downcase
|
209
|
-
return op if KNOWN_OPERATIONS.include?(op)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
|
213
|
-
|
214
|
-
def handle_exception_in_explain
|
215
|
-
yield
|
216
|
-
rescue => e
|
217
|
-
::TingYun::Agent.logger.error("Error getting query plan:", e)
|
218
|
-
nil
|
219
|
-
end
|
220
|
-
|
221
72
|
|
222
73
|
def get_connection(config, &connector)
|
223
|
-
ConnectionManager.instance.get_connection(config, &connector)
|
74
|
+
TingYun::Agent::Database::ConnectionManager.instance.get_connection(config, &connector)
|
224
75
|
end
|
225
76
|
|
226
77
|
def close_connections
|
227
|
-
ConnectionManager.instance.close_connections
|
78
|
+
TingYun::Agent::Database::ConnectionManager.instance.close_connections
|
228
79
|
end
|
229
80
|
|
230
|
-
# Returns a cached connection for a given ActiveRecord
|
231
|
-
# configuration - these are stored or reopened as needed, and if
|
232
|
-
# we cannot get one, we ignore it and move on without explaining
|
233
|
-
# the sql
|
234
|
-
class ConnectionManager
|
235
|
-
include Singleton
|
236
|
-
|
237
|
-
def get_connection(config, &connector)
|
238
|
-
@connections ||= {}
|
239
|
-
|
240
|
-
connection = @connections[config]
|
241
|
-
|
242
|
-
return connection if connection
|
243
81
|
|
244
|
-
begin
|
245
|
-
@connections[config] = connector.call(config)
|
246
|
-
rescue => e
|
247
|
-
::TingYun::Agent.logger.error("Caught exception trying to get connection to DB for explain.", e)
|
248
|
-
nil
|
249
|
-
end
|
250
|
-
end
|
251
82
|
|
252
|
-
|
253
|
-
def close_connections
|
254
|
-
@connections ||= {}
|
255
|
-
@connections.values.each do |connection|
|
256
|
-
begin
|
257
|
-
connection.disconnect!
|
258
|
-
rescue
|
259
|
-
end
|
260
|
-
end
|
83
|
+
RECORD_FOR = [:raw, :obfuscated].freeze
|
261
84
|
|
262
|
-
|
263
|
-
|
85
|
+
def should_record_sql?(key)
|
86
|
+
RECORD_FOR.include?(record_sql_method(key.to_sym))
|
264
87
|
end
|
265
88
|
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
@sql = TingYun::Agent::Database.capture_query(sql)
|
271
|
-
@config = config
|
272
|
-
@explainer = explainer
|
273
|
-
end
|
274
|
-
|
275
|
-
def adapter
|
276
|
-
config && config[:adapter]
|
277
|
-
end
|
89
|
+
def sql_sampler_enabled?
|
90
|
+
Agent.config[:'nbs.action_tracer.enabled'] &&
|
91
|
+
Agent.config[:'nbs.action_tracer.slow_sql'] &&
|
92
|
+
should_record_sql?('nbs.action_tracer.record_sql')
|
278
93
|
end
|
279
94
|
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
attr_reader :obfuscator
|
285
|
-
|
286
|
-
def initialize
|
287
|
-
reset
|
288
|
-
end
|
289
|
-
|
290
|
-
def reset
|
291
|
-
@obfuscator = method(:default_sql_obfuscator)
|
292
|
-
end
|
293
|
-
|
294
|
-
QUERY_TOO_LARGE_MESSAGE = "Query too large (over 16k characters) to safely obfuscate"
|
295
|
-
FAILED_TO_OBFUSCATE_MESSAGE = "Failed to obfuscate SQL query - quote characters remained after obfuscation"
|
296
|
-
|
297
|
-
def default_sql_obfuscator(sql)
|
298
|
-
stmt = sql.kind_of?(Statement) ? sql : Statement.new(sql)
|
299
|
-
|
300
|
-
if stmt.sql[-3,3] == '...'
|
301
|
-
return QUERY_TOO_LARGE_MESSAGE
|
302
|
-
end
|
303
|
-
|
304
|
-
obfuscate_double_quotes = stmt.adapter.to_s !~ /postgres|sqlite/
|
305
|
-
|
306
|
-
obfuscated = obfuscate_numeric_literals(stmt.sql)
|
307
|
-
|
308
|
-
if obfuscate_double_quotes
|
309
|
-
obfuscated = obfuscate_quoted_literals(obfuscated)
|
310
|
-
obfuscated = remove_comments(obfuscated)
|
311
|
-
if contains_quotes?(obfuscated)
|
312
|
-
obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
|
313
|
-
end
|
314
|
-
else
|
315
|
-
obfuscated = obfuscate_single_quote_literals(obfuscated)
|
316
|
-
obfuscated = remove_comments(obfuscated)
|
317
|
-
if contains_single_quotes?(obfuscated)
|
318
|
-
obfuscated = FAILED_TO_OBFUSCATE_MESSAGE
|
319
|
-
end
|
320
|
-
end
|
321
|
-
|
322
|
-
|
323
|
-
obfuscated.to_s # return back to a regular String
|
324
|
-
end
|
325
|
-
|
326
|
-
QUOTED_STRINGS_REGEX = /'(?:[^']|'')*'|"(?:[^"]|"")*"/
|
327
|
-
LABEL_LINE_REGEX = /^([^:\n]*:\s+).*$/.freeze
|
328
|
-
|
329
|
-
def obfuscate_postgres_explain(explain)
|
330
|
-
explain.gsub!(QUOTED_STRINGS_REGEX) do |match|
|
331
|
-
match.start_with?('"') ? match : '?'
|
332
|
-
end
|
333
|
-
explain.gsub!(LABEL_LINE_REGEX, '\1?')
|
334
|
-
explain
|
335
|
-
end
|
336
|
-
|
337
|
-
module ObfuscationHelpers
|
338
|
-
# Note that the following two regexes are applied to a reversed version
|
339
|
-
# of the query. This is why the backslash escape sequences (\' and \")
|
340
|
-
# appear reversed within them.
|
341
|
-
#
|
342
|
-
# Note that some database adapters (notably, PostgreSQL with
|
343
|
-
# standard_conforming_strings on and MySQL with NO_BACKSLASH_ESCAPES on)
|
344
|
-
# do not apply special treatment to backslashes within quoted string
|
345
|
-
# literals. We don't have an easy way of determining whether the
|
346
|
-
# database connection from which a query was captured was operating in
|
347
|
-
# one of these modes, but the obfuscation is done in such a way that it
|
348
|
-
# should not matter.
|
349
|
-
#
|
350
|
-
# Reversing the query string before obfuscation allows us to get around
|
351
|
-
# the fact that a \' appearing within a string may or may not terminate
|
352
|
-
# the string, because we know that a string cannot *start* with a \'.
|
353
|
-
REVERSE_SINGLE_QUOTES_REGEX = /'(?:''|'\\|[^'])*'/
|
354
|
-
REVERSE_ANY_QUOTES_REGEX = /'(?:''|'\\|[^'])*'|"(?:""|"\\|[^"])*"/
|
355
|
-
|
356
|
-
NUMERICS_REGEX = /\b\d+\b/
|
357
|
-
|
358
|
-
# We take a conservative, overly-aggressive approach to obfuscating
|
359
|
-
# comments, and drop everything from the query after encountering any
|
360
|
-
# character sequence that could be a comment initiator. We do this after
|
361
|
-
# removal of string literals to avoid accidentally over-obfuscating when
|
362
|
-
# a string literal contains a comment initiator.
|
363
|
-
SQL_COMMENT_REGEX = Regexp.new('(?:/\*|--|#).*', Regexp::MULTILINE).freeze
|
364
|
-
|
365
|
-
# We use these to check whether the query contains any quote characters
|
366
|
-
# after obfuscation. If so, that's a good indication that the original
|
367
|
-
# query was malformed, and so our obfuscation can't reliabily find
|
368
|
-
# literals. In such a case, we'll replace the entire query with a
|
369
|
-
# placeholder.
|
370
|
-
LITERAL_SINGLE_QUOTE = "'".freeze
|
371
|
-
LITERAL_DOUBLE_QUOTE = '"'.freeze
|
372
|
-
|
373
|
-
PLACEHOLDER = '?'.freeze
|
374
|
-
|
375
|
-
def obfuscate_single_quote_literals(sql)
|
376
|
-
obfuscated = sql.reverse
|
377
|
-
obfuscated.gsub!(REVERSE_SINGLE_QUOTES_REGEX, PLACEHOLDER)
|
378
|
-
obfuscated.reverse!
|
379
|
-
obfuscated
|
380
|
-
end
|
381
|
-
|
382
|
-
def obfuscate_quoted_literals(sql)
|
383
|
-
obfuscated = sql.reverse
|
384
|
-
obfuscated.gsub!(REVERSE_ANY_QUOTES_REGEX, PLACEHOLDER)
|
385
|
-
obfuscated.reverse!
|
386
|
-
obfuscated
|
387
|
-
end
|
388
|
-
|
389
|
-
def obfuscate_numeric_literals(sql)
|
390
|
-
sql.gsub(NUMERICS_REGEX, PLACEHOLDER)
|
391
|
-
end
|
392
|
-
|
393
|
-
def remove_comments(sql)
|
394
|
-
sql.gsub(SQL_COMMENT_REGEX, PLACEHOLDER)
|
395
|
-
end
|
396
|
-
|
397
|
-
def contains_single_quotes?(str)
|
398
|
-
str.include?(LITERAL_SINGLE_QUOTE)
|
399
|
-
end
|
400
|
-
|
401
|
-
def contains_quotes?(str)
|
402
|
-
str.include?(LITERAL_SINGLE_QUOTE) || str.include?(LITERAL_DOUBLE_QUOTE)
|
403
|
-
end
|
404
|
-
end
|
405
|
-
include ObfuscationHelpers
|
95
|
+
def should_action_collect_explain_plans?
|
96
|
+
should_record_sql?("nbs.action_tracer.record_sql") &&
|
97
|
+
Agent.config["nbs.action_tracer.explain_enabled".to_sym]
|
406
98
|
end
|
407
99
|
|
408
100
|
end
|