simple_apm 1.0.5 → 1.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 69c8c44cd6c1c1c98083753871364bbeb78f9f8d6c26d1d4a4bdd791e22349a3
4
- data.tar.gz: 507a53cdc6cb281eb76d765560638138010e26b125f515a9289b1a6472677e0f
3
+ metadata.gz: c6c7724ac3789dbc51e7e8204bda7db3f3bd6ab8b4902a6533a36e75c39f26bd
4
+ data.tar.gz: 70956572137d0930ad2e3e57f365bab309d33c208a85bd596c8a4669968e804c
5
5
  SHA512:
6
- metadata.gz: a04629e1d91833e080cf9b503379332fbd1e490855306fd93a20ef7700db0bb642959feec81356dfd8ea21cd9b22015e82f3a8c10cda045cc29cc3efd622e009
7
- data.tar.gz: 9033e87a25ecf8890edf0f5e56fb71d4875e8c304e4c5c85e73f3567952ac2bc9853f4807e0a736b281614029d4d32d9713b02b68a7b4c088bac050ce406fcc6
6
+ metadata.gz: 8fad79fd506a39a77f468fd48ab5da873466d340a04082bb33f124aaa3d2a300b853ff9b64db0f35ea4c3ee4a792a18923039fabfa04271e7eeedc43a55449ce
7
+ data.tar.gz: 1f71a63f8c3984a8f59f26261bbda98c5665bb8074d43eaa004be8e1ee9d827fb9ef934fd176db6c3a8de0d1edadec382bf9c3d6dce654de7caf00a40b521ce7
data/README.md CHANGED
@@ -1,4 +1,4 @@
1
- # SimpleApm
1
+ # SimpleApm (Rails Engine)
2
2
  基于Redis的简单的Web请求性能监控/慢事务追踪工具
3
3
 
4
4
  以天为维度记录:
@@ -7,6 +7,19 @@
7
7
  - 记录每个action的平均访问时间
8
8
  - 记录慢请求的详情和对应SQL详情(多余的会删掉)
9
9
  - 以10分钟为刻度记录平均/最慢访问时间、次数等性能指标,并生成图表
10
+ - 记录请求中外部http访问时间
11
+ - (TODO)记录Sidekiq的整套信息
12
+
13
+ ## 原理
14
+
15
+ 围绕Rack记录请求级别的相关信息,使用redis作为数据存储/计算工具来记录慢事务
16
+
17
+ 数据传递核心为:[Active Support Instrumentation](https://guides.rubyonrails.org/active_support_instrumentation.html)
18
+
19
+ 处理Instrument方式为开启一个不影响主线程的常驻线程,循环计算处理数据
20
+
21
+ 获取内存信息用到了gem: [get_process_mem](https://github.com/schneems/get_process_mem),经测试在linux系统耗时在1ms以下
22
+
10
23
 
11
24
  ## Usage
12
25
 
@@ -32,10 +45,6 @@ And then execute:
32
45
  $ bundle
33
46
  ```
34
47
 
35
- Or install it yourself as:
36
- ```bash
37
- $ gem install simple_apm
38
- ```
39
48
 
40
49
  ## Contributing
41
50
  Contribution directions go here.
@@ -24,6 +24,7 @@ module SimpleApm
24
24
  sec_str(request.during),
25
25
  sec_str(request.db_runtime),
26
26
  sec_str(request.view_runtime),
27
+ sec_str(request.net_http_during),
27
28
  request.memory_during.to_f.round(1),
28
29
  request.host,
29
30
  request.remote_addr
@@ -0,0 +1,35 @@
1
+ # 请求对应的SQL查询列表
2
+ module SimpleApm
3
+ class HttpRequest
4
+ attr_accessor :request_id, :url, :line, :filename, :during, :started, :path, :host, :name
5
+ def initialize(h, request = nil)
6
+ h.each do |k, v|
7
+ send("#{k}=", v) rescue puts "attr #{k} not set!"
8
+ end
9
+ @request = request
10
+ end
11
+
12
+ def request
13
+ @request ||= SimpleApm::Request.find(request_id)
14
+ end
15
+
16
+ class << self
17
+ # @return [Array<SimpleApm::HttpRequest>]
18
+ def find_by_request_id(request_id)
19
+ SimpleApm::Redis.lrange(key(request_id), 0, -1).map{|x|SimpleApm::HttpRequest.new JSON.parse(x)}
20
+ end
21
+
22
+ def delete_by_request_id(request_id)
23
+ SimpleApm::Redis.del(key(request_id))
24
+ end
25
+
26
+ def create(request_id, info)
27
+ SimpleApm::Redis.rpush(key(request_id), info.to_json)
28
+ end
29
+
30
+ def key(request_id)
31
+ SimpleApm::RedisKey["http_request:#{request_id}"]
32
+ end
33
+ end
34
+ end
35
+ end
@@ -5,13 +5,17 @@ module SimpleApm
5
5
  :during, :started, :db_runtime, :view_runtime,
6
6
  :controller, :action, :format, :method,
7
7
  :host, :remote_addr, :url, :completed_memory, :memory_during,
8
- :exception, :status
8
+ :exception, :status, :net_http_during
9
9
  def initialize(h)
10
10
  h.each do |k, v|
11
11
  send("#{k}=", v) rescue puts "attr #{k} not set!"
12
12
  end
13
13
  end
14
14
 
15
+ def net_http_requests
16
+ @requests ||= SimpleApm::HttpRequest.find_by_request_id(request_id)
17
+ end
18
+
15
19
  def sqls
16
20
  @sqls ||= SimpleApm::Sql.find_by_request_id(request_id)
17
21
  end
@@ -6,6 +6,7 @@
6
6
  <th>总耗时</th>
7
7
  <th>db_time</th>
8
8
  <th>view time</th>
9
+ <th>外部http time</th>
9
10
  <th>内存增幅(M)</th>
10
11
  <th>host</th>
11
12
  <th>remote_addr</th>
@@ -11,7 +11,9 @@
11
11
  <label>响应时间:</label>
12
12
  <span>
13
13
  <%= sec_str @request.during %>
14
- (数据库:<%= sec_str @request.db_runtime %>, View: <%= sec_str @request.view_runtime %>)
14
+ (数据库:<%= sec_str @request.db_runtime %>,
15
+ View: <%= sec_str @request.view_runtime %>,
16
+ 外部请求: <%= sec_str @request.net_http_during %>)
15
17
  </span>
16
18
  </p>
17
19
  <p>
@@ -32,42 +34,65 @@
32
34
  <% if @request.exception.present? && @request.exception != 'null' %>
33
35
  <p>
34
36
  <label>报错信息: </label>
35
- <pre><%= @request.exception %></pre>
37
+ <pre><%= @request.exception %></pre>
36
38
  </p>
37
39
  <% end %>
38
- <table id="sql-table" class="table">
39
- <thead>
40
- <tr>
41
- <td>time</td>
42
- <td>name</td>
43
- <td>sql</td>
44
- <td>时间(ms)</td>
45
- <td>位置</td>
46
- </tr>
47
- </thead>
48
- <tbody>
49
- <% @request.sqls.each do |sql| %>
40
+ <% if @request.net_http_requests.present? %>
41
+ <p>外部请求</p>
42
+ <table class="table table-bordered">
50
43
  <tr>
51
- <td><%= time_label sql.started %></td>
52
- <td>
53
- <%= sql.name %>
54
- </td>
55
- <td>
56
- <div class="sql"><%= sql.full_sql -%></div>
57
- </td>
58
- <td><%= sec_str sql.during %></td>
59
- <td><%= sql.filename %>:<%= sql.line %></td>
44
+ <th>开始时间</th>
45
+ <th>URL</th>
46
+ <th>耗时</th>
47
+ <th>其他信息</th>
48
+ </tr>
49
+ <% @request.net_http_requests.each do |r| %>
50
+ <tr>
51
+ <td><%= r.started %></td>
52
+ <td><%= r.url %></td>
53
+ <td><%= sec_str r.during %></td>
54
+ <td><%= "#{r.filename}: #{r.line}" if r.filename.present? %></td>
55
+ </tr>
56
+ <% end %>
57
+ </table>
58
+ <% end %>
59
+
60
+ <% if @request.sqls.present? %>
61
+ <p>SQL列表</p>
62
+ <table id="sql-table" class="table">
63
+ <thead>
64
+ <tr>
65
+ <td>time</td>
66
+ <td>name</td>
67
+ <td>sql</td>
68
+ <td>时间(ms)</td>
69
+ <td>位置</td>
60
70
  </tr>
61
- <% end %>
62
- </tbody>
63
- </table>
64
- <script type="text/javascript">
65
- $(document).ready(function(){
66
- $('#sql-table').DataTable({
67
- bPaginate: false,
68
- // 自定义的排序方法
69
- aoColumnDefs: [{sType: "sec", aTargets: [3]}]
70
- })
71
+ </thead>
72
+ <tbody>
73
+ <% @request.sqls.each do |sql| %>
74
+ <tr>
75
+ <td><%= time_label sql.started %></td>
76
+ <td>
77
+ <%= sql.name %>
78
+ </td>
79
+ <td>
80
+ <div class="sql"><%= sql.full_sql -%></div>
81
+ </td>
82
+ <td><%= sec_str sql.during %></td>
83
+ <td><%= sql.filename %>:<%= sql.line %></td>
84
+ </tr>
85
+ <% end %>
86
+ </tbody>
87
+ </table>
88
+ <script type="text/javascript">
89
+ $(document).ready(function () {
90
+ $('#sql-table').DataTable({
91
+ bPaginate: false,
92
+ // 自定义的排序方法
93
+ aoColumnDefs: [{sType: "sec", aTargets: [3]}]
94
+ })
71
95
 
72
- })
73
- </script>
96
+ })
97
+ </script>
98
+ <% end %>
data/lib/simple_apm.rb CHANGED
@@ -1,20 +1,45 @@
1
- require "simple_apm/setting"
2
- require "simple_apm/redis"
3
- require "simple_apm/engine"
1
+ require 'simple_apm/setting'
2
+ require 'simple_apm/redis'
3
+ require 'simple_apm/engine'
4
+ require 'simple_apm/worker'
5
+ require 'simple_apm/net_http'
4
6
  require 'callsite'
5
7
  require 'get_process_mem'
6
8
  module SimpleApm
7
- # 订阅log ---- start ----
9
+
10
+ SimpleApm::NetHttp.install
11
+
8
12
  ActiveSupport::Notifications.subscribe('process_action.action_controller') do |name, started, finished, unique_id, payload|
9
13
  ProcessingThread.add_event(
10
14
  name: name,
11
15
  request_id: Thread.current['action_dispatch.request_id'],
12
16
  started: started, finished: finished,
13
- payload: payload,
17
+ payload: payload.reject{|k,v|k.to_s=='headers'},
14
18
  started_memory: Thread.current[:current_process_memory],
15
- completed_memory: GetProcessMem.new.mb
19
+ completed_memory: GetProcessMem.new.mb,
20
+ net_http_during: Thread.current[:net_http_during]
16
21
  )
17
22
  Thread.current['action_dispatch.request_id'] = nil
23
+ Thread.current[:net_http_during] = nil
24
+ end
25
+
26
+ ActiveSupport::Notifications.subscribe 'net_http.request' do |name, started, finished, unique_id, payload|
27
+ th = Thread.current['action_dispatch.request_id'].present? ? Thread.current : Thread.main
28
+ request_id = th['action_dispatch.request_id']
29
+ if request_id
30
+ during = finished - started
31
+ th[:net_http_during] += during if th[:net_http_during]
32
+ if dev_caller = caller.detect {|c| c.include?(Rails.root.to_s) && !c.include?('/vendor/')}
33
+ c = ::Callsite.parse(dev_caller)
34
+ payload.merge!(:line => c.line, :filename => c.filename.to_s.gsub(Rails.root.to_s, ''), :method => c.method)
35
+ end
36
+ ProcessingThread.add_event(
37
+ name: name,
38
+ request_id: request_id,
39
+ started: started, finished: finished,
40
+ payload: payload
41
+ )
42
+ end
18
43
  end
19
44
 
20
45
  ActiveSupport::Notifications.subscribe 'sql.active_record' do |name, started, finished, unique_id, payload|
@@ -48,107 +73,13 @@ module SimpleApm
48
73
  Thread.current[:events] ||= []
49
74
  loop do
50
75
  while e = Thread.current[:events].shift
51
- Worker.process! e
76
+ ::SimpleApm::Worker.process! e
52
77
  end
53
- sleep 1
78
+ sleep 0.5
54
79
  end
55
80
  end
56
81
  end
57
82
  end
58
83
  end
59
84
 
60
- # 处理信息的操作方法
61
- class Worker
62
- class << self
63
- def process!(event)
64
- SimpleApm::RedisKey.query_date = nil
65
- if event[:name]=='process_action.action_controller'
66
- process_controller(event)
67
- elsif event[:name]=='sql.active_record'
68
- process_sql(event)
69
- end
70
- end
71
-
72
- private
73
- def process_controller(event)
74
- payload = event[:payload]
75
- started_memory = event[:started_memory]
76
- completed_memory = event[:completed_memory]
77
- finished = event[:finished]
78
- started = event[:started]
79
- request_id = event[:request_id]
80
- begin
81
- need_skip = payload[:controller] == 'SimpleApm::ApmController'
82
- need_skip = true if SimpleApm::Setting::EXCLUDE_ACTIONS.include?("#{payload[:controller]}##{payload[:action]}")
83
- need_skip = true if payload[:status].to_s=='302' && payload[:path].to_s=~/login/ && payload[:method].to_s.downcase=='get'
84
-
85
- if request_id.present?
86
- if need_skip
87
- SimpleApm::Sql.delete_by_request_id(request_id)
88
- else
89
- action_name = "#{payload[:controller]}##{payload[:action]}"
90
- memory_during = completed_memory - started_memory rescue 0
91
- info = {
92
- request_id: request_id,
93
- action_name: action_name,
94
- url: payload[:path],
95
- during: finished - started,
96
- started: started.to_s,
97
- db_runtime: payload[:db_runtime].to_f / 1000,
98
- view_runtime: payload[:view_runtime].to_f / 1000,
99
- controller: payload[:controller],
100
- action: payload[:action],
101
- host: Socket.gethostname,
102
- remote_addr: (payload[:headers]['HTTP_X_REAL_IP'] rescue nil),
103
- method: payload[:method],
104
- completed_memory: completed_memory,
105
- memory_during: memory_during,
106
- format: payload[:format],
107
- exception: payload[:exception].presence.to_json
108
- }.with_indifferent_access
109
- info[:status] = '500' if payload[:exception]
110
- # 存储
111
- in_slow = SimpleApm::SlowRequest.update_by_request info
112
- in_action_info = SimpleApm::Action.update_by_request info
113
- SimpleApm::Hit.update_by_request info
114
- if in_action_info || in_slow
115
- SimpleApm::Request.create info
116
- else
117
- SimpleApm::Sql.delete_by_request_id(request_id)
118
- end
119
- end
120
- end
121
- rescue => e
122
- Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
123
- end
124
- end
125
-
126
- def process_sql(event)
127
- payload = event[:payload]
128
- finished = event[:finished]
129
- started = event[:started]
130
- request_id = event[:request_id]
131
- begin
132
- during = finished - started
133
- if request_id.present? && ( SimpleApm::Setting::SQL_CRITICAL_TIME.blank? || during > SimpleApm::Setting::SQL_CRITICAL_TIME)
134
- # ActiveRecord::Relation::QueryAttribute
135
- sql_value = payload[:binds].map {|q| [q.name, q.value]} rescue nil
136
- info = {
137
- request_id: request_id,
138
- name: payload[:name],
139
- during: during,
140
- started: started,
141
- sql: payload[:sql],
142
- value: sql_value,
143
- filename: payload[:filename],
144
- line: payload[:line]
145
- }.with_indifferent_access
146
- SimpleApm::Sql.create request_id, info
147
- end
148
- rescue => e
149
- Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
150
- end
151
- end
152
- end
153
- end
154
85
  end
@@ -9,6 +9,7 @@ module SimpleApm
9
9
  SimpleApm::ProcessingThread.start!
10
10
  Thread.current['action_dispatch.request_id'] = env['action_dispatch.request_id']
11
11
  Thread.current[:current_process_memory] = GetProcessMem.new.mb
12
+ Thread.current[:net_http_during] = 0.0
12
13
  end
13
14
  @app.call(env)
14
15
  end
@@ -0,0 +1,19 @@
1
+ module SimpleApm
2
+ class NetHttp
3
+ class << self
4
+ def install
5
+ Net::HTTP.class_eval do
6
+ alias orig_request request unless method_defined?(:orig_request)
7
+
8
+ def request(req, body = nil, &block)
9
+ url = "http://#{@address}:#{@port}#{req.path}"
10
+ ActiveSupport::Notifications.instrument "net_http.request", url: url, host: @address, path: req.path do
11
+ @response = orig_request(req, body, &block)
12
+ end
13
+ @response
14
+ end
15
+ end
16
+ end
17
+ end
18
+ end
19
+ end
@@ -1,3 +1,3 @@
1
1
  module SimpleApm
2
- VERSION = '1.0.5'
2
+ VERSION = '1.0.6'
3
3
  end
@@ -0,0 +1,127 @@
1
+ # 处理信息的操作方法
2
+ module SimpleApm
3
+ class Worker
4
+ class << self
5
+ def process!(event)
6
+ SimpleApm::RedisKey.query_date = nil
7
+ if event[:name] == 'process_action.action_controller'
8
+ process_controller(event)
9
+ elsif event[:name] == 'sql.active_record'
10
+ process_sql(event)
11
+ elsif event[:name] == 'net_http.request'
12
+ process_http_request(event)
13
+ end
14
+ end
15
+
16
+ private
17
+
18
+ def process_controller(event)
19
+ payload = event[:payload]
20
+ started_memory = event[:started_memory]
21
+ completed_memory = event[:completed_memory]
22
+ finished = event[:finished]
23
+ started = event[:started]
24
+ request_id = event[:request_id]
25
+ net_http_during = event[:net_http_during]
26
+ begin
27
+ need_skip = payload[:controller] == 'SimpleApm::ApmController'
28
+ need_skip = true if SimpleApm::Setting::EXCLUDE_ACTIONS.include?("#{payload[:controller]}##{payload[:action]}")
29
+ need_skip = true if payload[:status].to_s == '302' && payload[:path].to_s =~ /login/ && payload[:method].to_s.downcase == 'get'
30
+
31
+ if request_id.present?
32
+ if need_skip
33
+ SimpleApm::Sql.delete_by_request_id(request_id)
34
+ else
35
+ action_name = "#{payload[:controller]}##{payload[:action]}"
36
+ memory_during = completed_memory - started_memory rescue 0
37
+ info = {
38
+ request_id: request_id,
39
+ action_name: action_name,
40
+ url: payload[:path],
41
+ during: finished - started,
42
+ started: started.to_s,
43
+ db_runtime: payload[:db_runtime].to_f / 1000,
44
+ view_runtime: payload[:view_runtime].to_f / 1000,
45
+ controller: payload[:controller],
46
+ action: payload[:action],
47
+ host: Socket.gethostname,
48
+ remote_addr: (payload[:headers]['HTTP_X_REAL_IP'] rescue nil),
49
+ method: payload[:method],
50
+ completed_memory: completed_memory,
51
+ memory_during: memory_during,
52
+ format: payload[:format],
53
+ net_http_during: net_http_during,
54
+ exception: payload[:exception].presence.to_json
55
+ }.with_indifferent_access
56
+ info[:status] = '500' if payload[:exception]
57
+ # 存储
58
+ in_slow = SimpleApm::SlowRequest.update_by_request info
59
+ in_action_info = SimpleApm::Action.update_by_request info
60
+ SimpleApm::Hit.update_by_request info
61
+ if in_action_info || in_slow
62
+ SimpleApm::Request.create info
63
+ else
64
+ SimpleApm::Sql.delete_by_request_id(request_id)
65
+ end
66
+ end
67
+ end
68
+ rescue => e
69
+ Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
70
+ end
71
+ end
72
+
73
+ def process_sql(event)
74
+ payload = event[:payload]
75
+ finished = event[:finished]
76
+ started = event[:started]
77
+ request_id = event[:request_id]
78
+ begin
79
+ during = finished - started
80
+ if request_id.present? && (SimpleApm::Setting::SQL_CRITICAL_TIME.blank? || during > SimpleApm::Setting::SQL_CRITICAL_TIME)
81
+ # ActiveRecord::Relation::QueryAttribute
82
+ sql_value = payload[:binds].map {|q| [q.name, q.value]} rescue nil
83
+ info = {
84
+ request_id: request_id,
85
+ name: payload[:name],
86
+ during: during,
87
+ started: started,
88
+ sql: payload[:sql],
89
+ value: sql_value,
90
+ filename: payload[:filename],
91
+ line: payload[:line]
92
+ }.with_indifferent_access
93
+ SimpleApm::Sql.create request_id, info
94
+ end
95
+ rescue => e
96
+ Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
97
+ end
98
+ end
99
+
100
+ def process_http_request(event)
101
+ payload = event[:payload]
102
+ finished = event[:finished]
103
+ started = event[:started]
104
+ request_id = event[:request_id]
105
+ begin
106
+ during = finished - started
107
+ if request_id.present?
108
+ info = {
109
+ request_id: request_id,
110
+ name: payload[:name],
111
+ during: during,
112
+ started: started,
113
+ url: payload[:url],
114
+ host: payload[:host],
115
+ path: payload[:path],
116
+ filename: payload[:filename],
117
+ line: payload[:line]
118
+ }.with_indifferent_access
119
+ SimpleApm::HttpRequest.create request_id, info
120
+ end
121
+ rescue => e
122
+ Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: simple_apm
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.5
4
+ version: 1.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - yuanyin.xia
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-07-26 00:00:00.000000000 Z
11
+ date: 2018-08-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rails
@@ -87,6 +87,7 @@ files:
87
87
  - app/models/simple_apm/action.rb
88
88
  - app/models/simple_apm/application_record.rb
89
89
  - app/models/simple_apm/hit.rb
90
+ - app/models/simple_apm/http_request.rb
90
91
  - app/models/simple_apm/request.rb
91
92
  - app/models/simple_apm/slow_request.rb
92
93
  - app/models/simple_apm/sql.rb
@@ -102,9 +103,11 @@ files:
102
103
  - lib/generators/simple_apm/install_generator.rb
103
104
  - lib/simple_apm.rb
104
105
  - lib/simple_apm/engine.rb
106
+ - lib/simple_apm/net_http.rb
105
107
  - lib/simple_apm/redis.rb
106
108
  - lib/simple_apm/setting.rb
107
109
  - lib/simple_apm/version.rb
110
+ - lib/simple_apm/worker.rb
108
111
  - lib/tasks/simple_apm_tasks.rake
109
112
  homepage: https://github.com/xiayuanyin/simple_apm
110
113
  licenses: