fluent-plugin-light-core 0.1.6 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84773035328d3e9f94b3a41242a24038c9f00bf84ff4e4c282ac67d44fa75350
-  data.tar.gz: 393720c2ce0fc4d55fabf7e9206ab57bf8085c0df9c5e043356cc87f356328c4
+  metadata.gz: af0f4865440862be4488030659ec9798b8eb9261bc2f9b95ee093a0b5de28387
+  data.tar.gz: 5a46ab336bb472b47f6572d5836e6fbbfeea8075e19dfa6a6781d4e938ceddb7
 SHA512:
-  metadata.gz: eca26afd55eb797b228c2cf65653882d31718c66630041ab05243c5be3d1b85f995a8dca5c8e6cd79971cb4db5735cd2d49ccb8f0e4c9357117e90252fae81ea
-  data.tar.gz: 8faa457619809fe3b26db9658fdf5dc66a960477541bc9ac7342b21612eb649743ec99be49ba1584e01fb8e615cd97f94dc9be9303d01fe222ed5ac654665226
+  metadata.gz: 045f2e53eba3fcf2ac10aa1a625e539bff38f6221c4a776d4c7a8b5250868e8bda1b0c1c59b07da1458c86787f07e6f9d568e4cb73562b7a4d261aaa0a42be1b
+  data.tar.gz: 7272c0c901fd28c3aec6b9241060f8dab6e64f3cc91f40ab7a044f866b951e4b3aa4cc4ac9428029914e7104ca43840abceee5b88cae3e0d0a95928ed337311c

data/Gemfile

@@ -1,3 +1,5 @@
 source "https://rubygems.org"
 
 gemspec
+
+gem "sentry-ruby"

data/fluent-plugin-light-core.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-light-core"
-  spec.version = "0.1.6"
+  spec.version = "0.2.1"
   spec.authors = ["LIN LI"]
   spec.email   = ["l.li@alphabets.cn"]
 
@@ -24,4 +24,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+  spec.add_runtime_dependency "sentry-ruby", [">= 4.1.5", "< 5"]
 end

data/lib/fluent/plugin/filter_light_core.rb

@@ -14,8 +14,8 @@
 # limitations under the License.
 
 require 'fluent/plugin/filter'
-require 'socket'
 require 'json'
+require 'sentry-ruby'
 
 module Fluent
   module Plugin
@@ -23,9 +23,8 @@ module Fluent
       Fluent::Plugin.register_filter('light_core', self)
 
       # 通知相关配置
-      config_param :notice, :bool,    default: false
-      config_param :port,   :integer, default: 7000
-      config_param :host,   :string,  default: '255.255.255.255'
+      config_param :sentry,     :bool,   default: false
+      config_param :sentry_dsn, :string, default: ''
 
       # Application 异常 settings 
       config_param :app_stream,  :string, default: 'stderr'
@@ -35,32 +34,35 @@ module Fluent
 
       # Nginx 异常 settings 
       config_param :lb_stream,  :string, default: 'stderr'
-      config_param :lb_code,    :array,  default: ['500'], value_type: :string
-      config_param :lb_elapsed, :float,  default: 3000
+      config_param :lb_code,    :array,  default: ['400', '500'], value_type: :string
+      config_param :lb_elapsed, :float,  default: 3
 
       # MongoDB 异常 settings 
       config_param :mongo_severity,  :array, default: ['F', 'E'], value_type: :string
       config_param :mongo_querytime, :float, default: 100
 
-      # 初始化UDP实例
+      # 初始化 Sentry
       def start
         super
 
-        if @notice
-          log.info('init udp connection')
-          @udp = UDPSocket.open()
-          @udp.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, 1)
+        if @sentry
+          log.info('init sentry')
+          Sentry.init do |config|
+            config.dsn = @sentry_dsn
+          
+            # To activate performance monitoring, set one of these options.
+            # We recommend adjusting the value in production:
+            config.traces_sample_rate = 1
+
+            # config.background_worker_threads = 2
+            config.transport.timeout = 10
+            config.transport.open_timeout = 10
+          end
         end
       end
 
-      # 清理UDP连接
+      # 清理
       def shutdown
-
-        if @notice
-          log.info('close udp connection')
-          @udp.close if @udp
-        end
-
         super
       end
 
@@ -88,11 +90,12 @@ module Fluent
 
       # Parse the application log
       def filter_app(tag, time, record)
-        file = record['file'].split('/').last.split('_')          # Parse log file name
-        log = record['log']                                       # Get detailed log content
+        file = record['file'].split('/').last.split('_')           # Parse log file name
+        log = record['log']                                        # Get detailed log content
 
         # Set common items
-        record['environment'] = Socket.gethostname.split('-')[0]  # dev | prd
+        # record['environment'] = Socket.gethostname.split('-')[0] # dev | prd
+        record['environment'] = ENV['FLUENTD_ENV']                 # dev | prd
         record['cid'] = file[0]                                    # container id
         record['cname'] = file[0].split('-')[1]                    # container name
         record['ctime'] = record['time']                           # container time
@@ -146,7 +149,7 @@ module Fluent
         file = record['file'].split('/').last.split('_')
         log = record['log']
 
-        record['environment'] = Socket.gethostname.split('-')[0]
+        record['environment'] = ENV['FLUENTD_ENV']
         record['cid'] = file[0]
         record['cname'] = tag
         record['ctime'] = record['time']
@@ -157,7 +160,7 @@ module Fluent
         
         # access log
         if /^[^ ]+ [^ ]+ [^ ]+ \[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{2}:\d{2}\]/.match(log)
-          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)" "(?<elapsed>[^\"]*)")?/.match(log)
+          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<elapsed>[^\"]*)" "(?<requestid>[^\"]*)" "(?<sessionid>[^\"]*)")?/.match(log)
 
           record['remote'] = item[:remote]
           record['host'] = item[:host]
@@ -169,8 +172,9 @@ module Fluent
           record['size'] = format_str(item[:size])
           record['referer'] = item[:referer]
           record['agent'] = item[:agent]
-          record['forwarder'] = item[:forwarder]
           record['elapsed'] = format_str(item[:elapsed])
+          record['requestid'] = item[:requestid]
+          record['sessionid'] = item[:sessionid]
 
           return record
         end
@@ -211,7 +215,7 @@ module Fluent
         file = record['file'].split('/').last.split('_')
         log = record['log']
 
-        record['environment'] = Socket.gethostname.split('-')[0]
+        record['environment'] = ENV['FLUENTD_ENV']
         record['cid'] = file[0]
         record['cname'] = tag
         record['ctime'] = record['time']
@@ -220,6 +224,7 @@ module Fluent
         record.delete('file')
         record.delete('time')
 
+        # 旧版本日志格式解析
         if /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4} [A-Z]/.match(log)
 
           item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) protocol:op_query (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
@@ -247,6 +252,30 @@ module Fluent
             end
           end
           
+          return record
+        else
+
+          # 版本4.4开始，默认日志为json格式
+          item = JSON.parse(log)
+          record['time'] = item['t']['$date']
+          record['severity'] = item['s']
+          record['component'] = item['c']
+          record['context'] = item['ctx']
+          record['identifier'] = item['id']
+          record['message'] = item['msg']
+
+          attributes = item['attr']
+          if attributes
+            record['querytime'] = attributes['durationMillis']
+            record['collection'] = attributes['ns']
+            record['command'] = attributes['command']
+            record['attr'] = attributes
+          end
+
+          record['tags'] = item['tags']
+          record['truncated'] = item['truncated']
+          record['size'] = item['size']
+        
           return record
         end
 
@@ -256,15 +285,18 @@ module Fluent
       # 确认是否发送通知
       def notice(tag, record)
 
-        unless @notice
+        unless @sentry
           return record
         end
 
         if tag == 'app'
+
+          # 未使用
           if @app_stream && record['stream']
             send(record) if record['stream'] == @app_stream
           end
 
+          # 未使用
           if @app_message.length > 0 && record['message']
             @app_message.each do |pattern|
               if pattern.match(record['message'])
@@ -275,11 +307,13 @@ module Fluent
           end
 
           if @app_status.length > 0 && record['status']
-            send(record) if @app_status.include?(record['status'])
+            message = 'Status code abnormal : ' + record['url']
+            send(tag, message, record) if @app_status.include?(record['status'])
           end
 
           if @app_elapsed > 0 && record['elapsed']
-            send(record) if record['elapsed'].to_f > @app_elapsed
+            message = 'Slow process : ' + record['url']
+            send(tag, message, record) if record['elapsed'].to_f >= @app_elapsed
           end
 
           return record
@@ -287,15 +321,17 @@ module Fluent
 
         if tag == 'lb'
           if @lb_stream && record['stream']
-            send(record) if record['stream'] == @lb_stream
+            send(tag, 'Stderror', record) if record['stream'] == @lb_stream
           end
 
           if @lb_code.length > 0 && record['code']
-            send(record) if @lb_code.include?(record['code'])
+            message = 'Status code abnormal : ' + record['path']
+            send(tag, message, record) if @lb_code.include?(record['code'])
           end
 
           if @lb_elapsed > 0 && record['elapsed']
-            send(record) if record['elapsed'].to_f > @lb_elapsed
+            message = 'Slow request : ' + record['path']
+            send(tag, message, record) if record['elapsed'].to_f >= @lb_elapsed
           end
     
           return record
@@ -303,11 +339,14 @@ module Fluent
 
         if tag == 'mongo'
           if @mongo_severity.length > 0 && record['severity']
-            send(record) if @mongo_severity.include?(record['severity'])
+            message = 'Severity level abnormal : ' + record['severity']
+            send(tag, message, record) if @mongo_severity.include?(record['severity'])
           end
 
           if @mongo_querytime > 0 && record['querytime']
-            send(record) if record['querytime'].to_f > @mongo_querytime
+            message = 'Slow query'
+            message = message + ' : ' + record['collection'] if record['collection']
+            send(tag, message, record) if record['querytime'].to_f >= @mongo_querytime
           end
 
           return record
@@ -318,17 +357,9 @@ module Fluent
       end
 
       # 发送UDP请求
-      def send(record)
-
-        log.info('send udp notice', record)
-
-        data = {
-          :query => '/api/log/notice',
-          :params => {
-            :data => record
-          }
-        }
-        @udp.send(data.to_json, 0, Socket.pack_sockaddr_in(@port, @host))
+      def send(tag, message, record)
+        log.debug('send udp notice')
+        Sentry.capture_message(message, :extra => record, :tags => {'log' => tag})
       end
 
       # 转数字

data/sample/README.md

@@ -32,3 +32,13 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
 
   % rm -f sample/*.pos
   % fluentd -c sample/source.conf -p lib/fluent/plugin
+
+## 添加 sentry 依赖
+
+- 编辑Gemfile
+
+  gem "sentry-ruby"
+
+- 安装依赖
+
+  gem install sentry-ruby

data/sample/source.conf

@@ -14,8 +14,8 @@
 
 <source>
   @type            tail
-  path             sample/lb*.log
-  pos_file         sample/source.lb.pos
+  path             sample/ingress-nginx*.log
+  pos_file         sample/source.ingress-nginx.pos
   tag              lb
   format           json
   read_from_head   true
@@ -79,6 +79,15 @@
 
 <filter **>
   @type            light_core
+  sentry           true
+  sentry_dsn       https://4447403a2a86437491d307ceb72898e5@sentry.zf.link/4
+  mongo_querytime  300
+  mongo_severity   F,E
+  lb_stream        stderr
+  lb_code          400,500
+  lb_elapsed       5
+  app_status       500
+  app_elapsed      5
 </filter>
 
 # <match app>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-light-core
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.1
 platform: ruby
 authors:
 - LIN LI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-20 00:00:00.000000000 Z
+date: 2021-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,6 +72,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: sentry-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.5
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 description: light core fluent plugin. support mongodb, nginx and application
 email:
 - l.li@alphabets.cn
@@ -87,6 +107,7 @@ files:
 - Rakefile
 - fluent-plugin-light-core.gemspec
 - lib/fluent/plugin/filter_light_core.rb
+- pkg/fluent-plugin-light-core-0.2.0.gem
 - sample/README.md
 - sample/source.conf
 - sample/udpsample.rb