simple_apm 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69c8c44cd6c1c1c98083753871364bbeb78f9f8d6c26d1d4a4bdd791e22349a3
-  data.tar.gz: 507a53cdc6cb281eb76d765560638138010e26b125f515a9289b1a6472677e0f
+  metadata.gz: c6c7724ac3789dbc51e7e8204bda7db3f3bd6ab8b4902a6533a36e75c39f26bd
+  data.tar.gz: 70956572137d0930ad2e3e57f365bab309d33c208a85bd596c8a4669968e804c
 SHA512:
-  metadata.gz: a04629e1d91833e080cf9b503379332fbd1e490855306fd93a20ef7700db0bb642959feec81356dfd8ea21cd9b22015e82f3a8c10cda045cc29cc3efd622e009
-  data.tar.gz: 9033e87a25ecf8890edf0f5e56fb71d4875e8c304e4c5c85e73f3567952ac2bc9853f4807e0a736b281614029d4d32d9713b02b68a7b4c088bac050ce406fcc6
+  metadata.gz: 8fad79fd506a39a77f468fd48ab5da873466d340a04082bb33f124aaa3d2a300b853ff9b64db0f35ea4c3ee4a792a18923039fabfa04271e7eeedc43a55449ce
+  data.tar.gz: 1f71a63f8c3984a8f59f26261bbda98c5665bb8074d43eaa004be8e1ee9d827fb9ef934fd176db6c3a8de0d1edadec382bf9c3d6dce654de7caf00a40b521ce7

data/README.md

@@ -1,4 +1,4 @@
-# SimpleApm
+# SimpleApm (Rails Engine)
 基于Redis的简单的Web请求性能监控/慢事务追踪工具
 
 以天为维度记录：
@@ -7,6 +7,19 @@
 - 记录每个action的平均访问时间
 - 记录慢请求的详情和对应SQL详情（多余的会删掉）
 - 以10分钟为刻度记录平均/最慢访问时间、次数等性能指标，并生成图表
+- 记录请求中外部http访问时间
+- (TODO)记录Sidekiq的整套信息
+
+## 原理
+
+围绕Rack记录请求级别的相关信息，使用redis作为数据存储/计算工具来记录慢事务
+
+数据传递核心为：[Active Support Instrumentation](https://guides.rubyonrails.org/active_support_instrumentation.html)
+
+处理Instrument方式为开启一个不影响主线程的常驻线程，循环计算处理数据
+
+获取内存信息用到了gem: [get_process_mem](https://github.com/schneems/get_process_mem)，经测试在linux系统耗时在1ms以下
+
 
 ## Usage
 
@@ -32,10 +45,6 @@ And then execute:
 $ bundle
 ```
 
-Or install it yourself as:
-```bash
-$ gem install simple_apm
-```
 
 ## Contributing
 Contribution directions go here.

data/app/controllers/simple_apm/apm_controller.rb

@@ -24,6 +24,7 @@ module SimpleApm
               sec_str(request.during),
               sec_str(request.db_runtime),
               sec_str(request.view_runtime),
+              sec_str(request.net_http_during),
               request.memory_during.to_f.round(1),
               request.host,
               request.remote_addr

data/app/models/simple_apm/http_request.rb

@@ -0,0 +1,35 @@
+# 请求对应的SQL查询列表
+module SimpleApm
+  class HttpRequest
+    attr_accessor :request_id, :url, :line, :filename, :during, :started, :path, :host, :name
+    def initialize(h, request = nil)
+      h.each do |k, v|
+        send("#{k}=", v)  rescue puts "attr #{k} not set!"
+      end
+      @request = request
+    end
+
+    def request
+      @request ||= SimpleApm::Request.find(request_id)
+    end
+
+    class << self
+      # @return [Array<SimpleApm::HttpRequest>]
+      def find_by_request_id(request_id)
+        SimpleApm::Redis.lrange(key(request_id), 0, -1).map{|x|SimpleApm::HttpRequest.new JSON.parse(x)}
+      end
+
+      def delete_by_request_id(request_id)
+        SimpleApm::Redis.del(key(request_id))
+      end
+
+      def create(request_id, info)
+        SimpleApm::Redis.rpush(key(request_id), info.to_json)
+      end
+
+      def key(request_id)
+        SimpleApm::RedisKey["http_request:#{request_id}"]
+      end
+    end
+  end
+end

data/app/models/simple_apm/request.rb

@@ -5,13 +5,17 @@ module SimpleApm
                   :during, :started, :db_runtime, :view_runtime,
                   :controller, :action, :format, :method,
                   :host, :remote_addr, :url, :completed_memory, :memory_during,
-                  :exception, :status
+                  :exception, :status, :net_http_during
     def initialize(h)
       h.each do |k, v|
         send("#{k}=", v) rescue puts "attr #{k} not set!"
       end
     end
 
+    def net_http_requests
+      @requests ||= SimpleApm::HttpRequest.find_by_request_id(request_id)
+    end
+
     def sqls
       @sqls ||= SimpleApm::Sql.find_by_request_id(request_id)
     end

data/app/views/simple_apm/apm/index.html.erb

@@ -6,6 +6,7 @@
     <th>总耗时</th>
     <th>db_time</th>
     <th>view time</th>
+    <th>外部http time</th>
     <th>内存增幅(M)</th>
     <th>host</th>
     <th>remote_addr</th>

data/app/views/simple_apm/apm/show.html.erb

@@ -11,7 +11,9 @@
   <label>响应时间：</label>
   <span>
   <%= sec_str @request.during %>
-    （数据库：<%= sec_str @request.db_runtime %>, View: <%= sec_str @request.view_runtime %>）
+    （数据库：<%= sec_str @request.db_runtime %>,
+    View: <%= sec_str @request.view_runtime %>,
+    外部请求: <%= sec_str @request.net_http_during %>）
 </span>
 </p>
 <p>
@@ -32,42 +34,65 @@
 <% if @request.exception.present? && @request.exception != 'null' %>
   <p>
     <label>报错信息: </label>
-    <pre><%= @request.exception %></pre>
+  <pre><%= @request.exception %></pre>
   </p>
 <% end %>
-<table id="sql-table" class="table">
-  <thead>
-  <tr>
-    <td>time</td>
-    <td>name</td>
-    <td>sql</td>
-    <td>时间(ms)</td>
-    <td>位置</td>
-  </tr>
-  </thead>
-  <tbody>
-  <% @request.sqls.each do |sql| %>
+<% if @request.net_http_requests.present? %>
+  <p>外部请求</p>
+  <table class="table table-bordered">
     <tr>
-      <td><%= time_label sql.started %></td>
-      <td>
-          <%= sql.name %>
-      </td>
-      <td>
-        <div class="sql"><%= sql.full_sql -%></div>
-      </td>
-      <td><%= sec_str sql.during %></td>
-      <td><%= sql.filename %>:<%= sql.line %></td>
+      <th>开始时间</th>
+      <th>URL</th>
+      <th>耗时</th>
+      <th>其他信息</th>
+    </tr>
+    <% @request.net_http_requests.each do |r| %>
+      <tr>
+        <td><%= r.started %></td>
+        <td><%= r.url %></td>
+        <td><%= sec_str r.during %></td>
+        <td><%= "#{r.filename}: #{r.line}" if r.filename.present? %></td>
+      </tr>
+    <% end %>
+  </table>
+<% end %>
+
+<% if @request.sqls.present? %>
+  <p>SQL列表</p>
+  <table id="sql-table" class="table">
+    <thead>
+    <tr>
+      <td>time</td>
+      <td>name</td>
+      <td>sql</td>
+      <td>时间(ms)</td>
+      <td>位置</td>
     </tr>
-  <% end %>
-  </tbody>
-</table>
-<script type="text/javascript">
-  $(document).ready(function(){
-    $('#sql-table').DataTable({
-      bPaginate: false,
-      // 自定义的排序方法
-      aoColumnDefs: [{sType: "sec", aTargets: [3]}]
-    })
+    </thead>
+    <tbody>
+    <% @request.sqls.each do |sql| %>
+      <tr>
+        <td><%= time_label sql.started %></td>
+        <td>
+          <%= sql.name %>
+        </td>
+        <td>
+          <div class="sql"><%= sql.full_sql -%></div>
+        </td>
+        <td><%= sec_str sql.during %></td>
+        <td><%= sql.filename %>:<%= sql.line %></td>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+  <script type="text/javascript">
+      $(document).ready(function () {
+          $('#sql-table').DataTable({
+              bPaginate: false,
+              // 自定义的排序方法
+              aoColumnDefs: [{sType: "sec", aTargets: [3]}]
+          })
 
-  })
-</script>
+      })
+  </script>
+<% end %>

data/lib/simple_apm.rb

@@ -1,20 +1,45 @@
-require "simple_apm/setting"
-require "simple_apm/redis"
-require "simple_apm/engine"
+require 'simple_apm/setting'
+require 'simple_apm/redis'
+require 'simple_apm/engine'
+require 'simple_apm/worker'
+require 'simple_apm/net_http'
 require 'callsite'
 require 'get_process_mem'
 module SimpleApm
-  # 订阅log ---- start ----
+
+  SimpleApm::NetHttp.install
+
   ActiveSupport::Notifications.subscribe('process_action.action_controller') do |name, started, finished, unique_id, payload|
     ProcessingThread.add_event(
         name: name,
         request_id: Thread.current['action_dispatch.request_id'],
         started: started, finished: finished,
-        payload: payload,
+        payload: payload.reject{|k,v|k.to_s=='headers'},
         started_memory: Thread.current[:current_process_memory],
-        completed_memory: GetProcessMem.new.mb
+        completed_memory: GetProcessMem.new.mb,
+        net_http_during: Thread.current[:net_http_during]
     )
     Thread.current['action_dispatch.request_id'] = nil
+    Thread.current[:net_http_during] = nil
+  end
+
+  ActiveSupport::Notifications.subscribe 'net_http.request' do |name, started, finished, unique_id, payload|
+    th = Thread.current['action_dispatch.request_id'].present? ? Thread.current : Thread.main
+    request_id = th['action_dispatch.request_id']
+    if request_id
+      during = finished - started
+      th[:net_http_during] += during if th[:net_http_during]
+      if dev_caller = caller.detect {|c| c.include?(Rails.root.to_s) && !c.include?('/vendor/')}
+        c = ::Callsite.parse(dev_caller)
+        payload.merge!(:line => c.line, :filename => c.filename.to_s.gsub(Rails.root.to_s, ''), :method => c.method)
+      end
+      ProcessingThread.add_event(
+          name: name,
+          request_id: request_id,
+          started: started, finished: finished,
+          payload: payload
+      )
+    end
   end
 
   ActiveSupport::Notifications.subscribe 'sql.active_record' do |name, started, finished, unique_id, payload|
@@ -48,107 +73,13 @@ module SimpleApm
           Thread.current[:events] ||= []
           loop do
             while e = Thread.current[:events].shift
-              Worker.process! e
+              ::SimpleApm::Worker.process! e
             end
-            sleep 1
+            sleep 0.5
           end
         end
       end
     end
   end
 
-  # 处理信息的操作方法
-  class Worker
-    class << self
-      def process!(event)
-        SimpleApm::RedisKey.query_date = nil
-        if event[:name]=='process_action.action_controller'
-          process_controller(event)
-        elsif event[:name]=='sql.active_record'
-          process_sql(event)
-        end
-      end
-
-      private
-      def process_controller(event)
-        payload = event[:payload]
-        started_memory = event[:started_memory]
-        completed_memory = event[:completed_memory]
-        finished = event[:finished]
-        started = event[:started]
-        request_id = event[:request_id]
-        begin
-          need_skip = payload[:controller] == 'SimpleApm::ApmController'
-          need_skip = true if SimpleApm::Setting::EXCLUDE_ACTIONS.include?("#{payload[:controller]}##{payload[:action]}")
-          need_skip = true if payload[:status].to_s=='302' && payload[:path].to_s=~/login/ && payload[:method].to_s.downcase=='get'
-
-          if request_id.present?
-            if need_skip
-              SimpleApm::Sql.delete_by_request_id(request_id)
-            else
-              action_name = "#{payload[:controller]}##{payload[:action]}"
-              memory_during = completed_memory - started_memory rescue 0
-              info = {
-                  request_id: request_id,
-                  action_name: action_name,
-                  url: payload[:path],
-                  during: finished - started,
-                  started: started.to_s,
-                  db_runtime: payload[:db_runtime].to_f / 1000,
-                  view_runtime: payload[:view_runtime].to_f / 1000,
-                  controller: payload[:controller],
-                  action: payload[:action],
-                  host: Socket.gethostname,
-                  remote_addr: (payload[:headers]['HTTP_X_REAL_IP'] rescue nil),
-                  method: payload[:method],
-                  completed_memory: completed_memory,
-                  memory_during: memory_during,
-                  format: payload[:format],
-                  exception: payload[:exception].presence.to_json
-              }.with_indifferent_access
-              info[:status] = '500' if payload[:exception]
-              # 存储
-              in_slow = SimpleApm::SlowRequest.update_by_request info
-              in_action_info = SimpleApm::Action.update_by_request info
-              SimpleApm::Hit.update_by_request info
-              if in_action_info || in_slow
-                SimpleApm::Request.create info
-              else
-                SimpleApm::Sql.delete_by_request_id(request_id)
-              end
-            end
-          end
-        rescue => e
-          Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
-        end
-      end
-
-      def process_sql(event)
-        payload = event[:payload]
-        finished = event[:finished]
-        started = event[:started]
-        request_id = event[:request_id]
-        begin
-          during = finished - started
-          if request_id.present? && ( SimpleApm::Setting::SQL_CRITICAL_TIME.blank? || during > SimpleApm::Setting::SQL_CRITICAL_TIME)
-            # ActiveRecord::Relation::QueryAttribute
-            sql_value = payload[:binds].map {|q| [q.name, q.value]} rescue nil
-            info = {
-                request_id: request_id,
-                name: payload[:name],
-                during: during,
-                started: started,
-                sql: payload[:sql],
-                value: sql_value,
-                filename: payload[:filename],
-                line: payload[:line]
-            }.with_indifferent_access
-            SimpleApm::Sql.create request_id, info
-          end
-        rescue => e
-          Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
-        end
-      end
-    end
-  end
 end

data/lib/simple_apm/engine.rb

@@ -9,6 +9,7 @@ module SimpleApm
         SimpleApm::ProcessingThread.start!
         Thread.current['action_dispatch.request_id'] = env['action_dispatch.request_id']
         Thread.current[:current_process_memory] = GetProcessMem.new.mb
+        Thread.current[:net_http_during] = 0.0
       end
       @app.call(env)
     end

data/lib/simple_apm/net_http.rb

@@ -0,0 +1,19 @@
+module SimpleApm
+  class NetHttp
+    class << self
+      def install
+        Net::HTTP.class_eval do
+          alias orig_request request unless method_defined?(:orig_request)
+
+          def request(req, body = nil, &block)
+            url = "http://#{@address}:#{@port}#{req.path}"
+            ActiveSupport::Notifications.instrument "net_http.request", url: url, host: @address, path: req.path do
+              @response = orig_request(req, body, &block)
+            end
+            @response
+          end
+        end
+      end
+    end
+  end
+end

data/lib/simple_apm/version.rb

@@ -1,3 +1,3 @@
 module SimpleApm
-  VERSION = '1.0.5'
+  VERSION = '1.0.6'
 end

data/lib/simple_apm/worker.rb

@@ -0,0 +1,127 @@
+# 处理信息的操作方法
+module SimpleApm
+  class Worker
+    class << self
+      def process!(event)
+        SimpleApm::RedisKey.query_date = nil
+        if event[:name] == 'process_action.action_controller'
+          process_controller(event)
+        elsif event[:name] == 'sql.active_record'
+          process_sql(event)
+        elsif event[:name] == 'net_http.request'
+          process_http_request(event)
+        end
+      end
+
+      private
+
+      def process_controller(event)
+        payload = event[:payload]
+        started_memory = event[:started_memory]
+        completed_memory = event[:completed_memory]
+        finished = event[:finished]
+        started = event[:started]
+        request_id = event[:request_id]
+        net_http_during = event[:net_http_during]
+        begin
+          need_skip = payload[:controller] == 'SimpleApm::ApmController'
+          need_skip = true if SimpleApm::Setting::EXCLUDE_ACTIONS.include?("#{payload[:controller]}##{payload[:action]}")
+          need_skip = true if payload[:status].to_s == '302' && payload[:path].to_s =~ /login/ && payload[:method].to_s.downcase == 'get'
+
+          if request_id.present?
+            if need_skip
+              SimpleApm::Sql.delete_by_request_id(request_id)
+            else
+              action_name = "#{payload[:controller]}##{payload[:action]}"
+              memory_during = completed_memory - started_memory rescue 0
+              info = {
+                  request_id: request_id,
+                  action_name: action_name,
+                  url: payload[:path],
+                  during: finished - started,
+                  started: started.to_s,
+                  db_runtime: payload[:db_runtime].to_f / 1000,
+                  view_runtime: payload[:view_runtime].to_f / 1000,
+                  controller: payload[:controller],
+                  action: payload[:action],
+                  host: Socket.gethostname,
+                  remote_addr: (payload[:headers]['HTTP_X_REAL_IP'] rescue nil),
+                  method: payload[:method],
+                  completed_memory: completed_memory,
+                  memory_during: memory_during,
+                  format: payload[:format],
+                  net_http_during: net_http_during,
+                  exception: payload[:exception].presence.to_json
+              }.with_indifferent_access
+              info[:status] = '500' if payload[:exception]
+              # 存储
+              in_slow = SimpleApm::SlowRequest.update_by_request info
+              in_action_info = SimpleApm::Action.update_by_request info
+              SimpleApm::Hit.update_by_request info
+              if in_action_info || in_slow
+                SimpleApm::Request.create info
+              else
+                SimpleApm::Sql.delete_by_request_id(request_id)
+              end
+            end
+          end
+        rescue => e
+          Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
+        end
+      end
+
+      def process_sql(event)
+        payload = event[:payload]
+        finished = event[:finished]
+        started = event[:started]
+        request_id = event[:request_id]
+        begin
+          during = finished - started
+          if request_id.present? && (SimpleApm::Setting::SQL_CRITICAL_TIME.blank? || during > SimpleApm::Setting::SQL_CRITICAL_TIME)
+            # ActiveRecord::Relation::QueryAttribute
+            sql_value = payload[:binds].map {|q| [q.name, q.value]} rescue nil
+            info = {
+                request_id: request_id,
+                name: payload[:name],
+                during: during,
+                started: started,
+                sql: payload[:sql],
+                value: sql_value,
+                filename: payload[:filename],
+                line: payload[:line]
+            }.with_indifferent_access
+            SimpleApm::Sql.create request_id, info
+          end
+        rescue => e
+          Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
+        end
+      end
+
+      def process_http_request(event)
+        payload = event[:payload]
+        finished = event[:finished]
+        started = event[:started]
+        request_id = event[:request_id]
+        begin
+          during = finished - started
+          if request_id.present?
+            info = {
+                request_id: request_id,
+                name: payload[:name],
+                during: during,
+                started: started,
+                url: payload[:url],
+                host: payload[:host],
+                path: payload[:path],
+                filename: payload[:filename],
+                line: payload[:line]
+            }.with_indifferent_access
+            SimpleApm::HttpRequest.create request_id, info
+          end
+        rescue => e
+          Logger.new("#{Rails.root}/log/simple_apm.log").info e.backtrace.join("\n")
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: simple_apm
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - yuanyin.xia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-26 00:00:00.000000000 Z
+date: 2018-08-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -87,6 +87,7 @@ files:
 - app/models/simple_apm/action.rb
 - app/models/simple_apm/application_record.rb
 - app/models/simple_apm/hit.rb
+- app/models/simple_apm/http_request.rb
 - app/models/simple_apm/request.rb
 - app/models/simple_apm/slow_request.rb
 - app/models/simple_apm/sql.rb
@@ -102,9 +103,11 @@ files:
 - lib/generators/simple_apm/install_generator.rb
 - lib/simple_apm.rb
 - lib/simple_apm/engine.rb
+- lib/simple_apm/net_http.rb
 - lib/simple_apm/redis.rb
 - lib/simple_apm/setting.rb
 - lib/simple_apm/version.rb
+- lib/simple_apm/worker.rb
 - lib/tasks/simple_apm_tasks.rake
 homepage: https://github.com/xiayuanyin/simple_apm
 licenses: