fluent-plugin-light-core 0.1.1 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c42d7ebbc40db04faa3683d2f19cc13b8faaad30f0a103dcd8284015cf41634
-  data.tar.gz: 7b1dc8dc732a0cdc15a8e9ac124a86a9981dd8a032488b73a6d2f00dc022b229
+  metadata.gz: 84773035328d3e9f94b3a41242a24038c9f00bf84ff4e4c282ac67d44fa75350
+  data.tar.gz: 393720c2ce0fc4d55fabf7e9206ab57bf8085c0df9c5e043356cc87f356328c4
 SHA512:
-  metadata.gz: 27d1c95d1b34923806055fc51e72f16e13af8c33d6c501aff6b2ad829c65817cfaacfce0ef687a373656738051035478cb4d06d6327666b8fa6adf872192a4d7
-  data.tar.gz: feb76078202bf213d7692ec0cd2a7faef62ce46219e4165ded55e13e9bc3eaaaf43f8de4eb49a5407c926ee6b091bd435fa7f4ce4f1aba5f2b12bc2507c517ad
+  metadata.gz: eca26afd55eb797b228c2cf65653882d31718c66630041ab05243c5be3d1b85f995a8dca5c8e6cd79971cb4db5735cd2d49ccb8f0e4c9357117e90252fae81ea
+  data.tar.gz: 8faa457619809fe3b26db9658fdf5dc66a960477541bc9ac7342b21612eb649743ec99be49ba1584e01fb8e615cd97f94dc9be9303d01fe222ed5ac654665226

data/fluent-plugin-light-core.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-light-core"
-  spec.version = "0.1.1"
+  spec.version = "0.1.6"
   spec.authors = ["LIN LI"]
   spec.email   = ["l.li@alphabets.cn"]
 

data/lib/fluent/plugin/filter_light_core.rb

@@ -13,25 +13,73 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "fluent/plugin/filter"
+require 'fluent/plugin/filter'
+require 'socket'
+require 'json'
 
 module Fluent
   module Plugin
     class LightCoreFilter < Fluent::Plugin::Filter
-      Fluent::Plugin.register_filter("light_core", self)
+      Fluent::Plugin.register_filter('light_core', self)
+
+      # 通知相关配置
+      config_param :notice, :bool,    default: false
+      config_param :port,   :integer, default: 7000
+      config_param :host,   :string,  default: '255.255.255.255'
+
+      # Application 异常 settings 
+      config_param :app_stream,  :string, default: 'stderr'
+      config_param :app_message, :array,  default: [],      value_type: :regexp
+      config_param :app_status,  :array,  default: ['500'], value_type: :string
+      config_param :app_elapsed, :float,  default: 2000
+
+      # Nginx 异常 settings 
+      config_param :lb_stream,  :string, default: 'stderr'
+      config_param :lb_code,    :array,  default: ['500'], value_type: :string
+      config_param :lb_elapsed, :float,  default: 3000
+
+      # MongoDB 异常 settings 
+      config_param :mongo_severity,  :array, default: ['F', 'E'], value_type: :string
+      config_param :mongo_querytime, :float, default: 100
+
+      # 初始化UDP实例
+      def start
+        super
+
+        if @notice
+          log.info('init udp connection')
+          @udp = UDPSocket.open()
+          @udp.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, 1)
+        end
+      end
+
+      # 清理UDP连接
+      def shutdown
+
+        if @notice
+          log.info('close udp connection')
+          @udp.close if @udp
+        end
+
+        super
+      end
 
+      # 主处理
       def filter(tag, time, record)
 
-        if tag == 'app'
-          return filter_app(tag, time, record)
+        if ['app', 'service'].include? tag
+          record = filter_app(tag, time, record)
+          return notice('app', record)
         end
 
         if ['lb', 'hub'].include? tag
-          return filter_lb(tag, time, record)
+          record = filter_lb(tag, time, record)
+          return notice('lb', record)
         end
 
         if ['mongo', 'secondary', 'arbiter'].include? tag
-          return filter_mongo(tag, time, record)
+          record = filter_mongo(tag, time, record)
+          return notice('mongo', record)
         end
 
         record
@@ -45,9 +93,9 @@ module Fluent
 
         # Set common items
         record['environment'] = Socket.gethostname.split('-')[0]  # dev | prd
-        record[:cid] = file[0]                                    # container id
-        record[:cname] = file[0].split('-')[1]                    # container name
-        record[:ctime] = record['time']                           # container time
+        record['cid'] = file[0]                                    # container id
+        record['cname'] = file[0].split('-')[1]                    # container name
+        record['ctime'] = record['time']                           # container time
 
         # Delete useless content
         record.delete('log')
@@ -59,12 +107,12 @@ module Fluent
 
           item = /^\[(?<time>[^\]]*)\] \[(?<component>[^\]]*)\] \[(?<levle>[^\]]*)\] (?<line>[^\ ]*) - - (?<uid>[^\ ]*) - (?<message>.*)$/.match(log)
 
-          record[:time] = item[:time]
-          record[:component] = item[:component]
-          record[:levle] = item[:levle]
-          record[:line] = item[:line]
-          record[:uid] = item[:uid]
-          record[:message] = item[:message]
+          record['time'] = item[:time]
+          record['component'] = item[:component]
+          record['levle'] = item[:levle]
+          record['line'] = item[:line]
+          record['uid'] = item[:uid]
+          record['message'] = item[:message]
 
           return record
         end
@@ -74,21 +122,21 @@ module Fluent
 
           item = /^\[(?<time>[^\]]*)\] \[(?<component>[^\]]*)\] \[(?<method>[^\]]*)\] (?<url>[^\ ]*) (?<status>[^\ ]*) (?<size>[^\ ]*) (?<uid>[^\ ]*) (?<elapsed>[^\ ]*) (?<addr>[^\ ]*)$/.match(log)
 
-          record[:time] = item[:time]
-          record[:component] = item[:component]
-          record[:method] = item[:method]
-          record[:url] = item[:url]
-          record[:status] = item[:status]
-          record[:size] = item[:size]
-          record[:uid] = item[:uid]
-          record[:elapsed] = item[:elapsed]
-          record[:addr] = item[:addr].gsub(/\n$/, '')
+          record['time'] = item[:time]
+          record['component'] = item[:component]
+          record['method'] = item[:method]
+          record['url'] = item[:url]
+          record['status'] = item[:status]
+          record['size'] = format_str(item[:size])
+          record['uid'] = item[:uid]
+          record['elapsed'] = format_str(item[:elapsed])
+          record['addr'] = item[:addr].gsub(/\n$/, '')
 
           return record
         end
 
         # Console log
-        record[:message] = log.gsub(/\n$/, '')
+        record['message'] = log.gsub(/\n$/, '')
         record
       end
 
@@ -99,9 +147,9 @@ module Fluent
         log = record['log']
 
         record['environment'] = Socket.gethostname.split('-')[0]
-        record[:cid] = file[0]
-        record[:cname] = tag
-        record[:ctime] = record['time']
+        record['cid'] = file[0]
+        record['cname'] = tag
+        record['ctime'] = record['time']
 
         record.delete('log')
         record.delete('file')
@@ -109,19 +157,20 @@ module Fluent
         
         # access log
         if /^[^ ]+ [^ ]+ [^ ]+ \[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{2}:\d{2}\]/.match(log)
-          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)")?/.match(log)
-
-          record[:remote] = item[:remote]
-          record[:host] = item[:host]
-          record[:user] = item[:user]
-          record[:time] = item[:time]
-          record[:method] = item[:method]
-          record[:path] = item[:path]
-          record[:code] = item[:code]
-          record[:size] = item[:size]
-          record[:referer] = item[:referer]
-          record[:agent] = item[:agent]
-          record[:forwarder] = item[:forwarder]
+          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)" "(?<elapsed>[^\"]*)")?/.match(log)
+
+          record['remote'] = item[:remote]
+          record['host'] = item[:host]
+          record['user'] = item[:user]
+          record['time'] = item[:time]
+          record['method'] = item[:method]
+          record['path'] = item[:path]
+          record['code'] = item[:code]
+          record['size'] = format_str(item[:size])
+          record['referer'] = item[:referer]
+          record['agent'] = item[:agent]
+          record['forwarder'] = item[:forwarder]
+          record['elapsed'] = format_str(item[:elapsed])
 
           return record
         end
@@ -130,29 +179,29 @@ module Fluent
         if /^\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2} \[error\]/.match(log)
           item = /^(?<time>[^ ]+ [^ ]+) \[(?<level>.*)\] (?<pid>\d*)#(?<tid>[^:]*): \*(?<cid>\d*) (?<message>.*)$/.match(log)
 
-          record[:time] = item[:time]
-          record[:level] = item[:level]
-          record[:message] = item[:message]
-          record[:process] = item[:pid]     # process id
-          record[:thread] = item[:tid]      # thread id
-          record[:counter] = item[:cid]     # counter
+          record['time'] = item[:time]
+          record['level'] = item[:level]
+          record['message'] = item[:message]
+          record['process'] = item[:pid]     # process id
+          record['thread'] = item[:tid]      # thread id
+          record['counter'] = item[:cid]     # counter
 
           detail = /request: "(?<method>[^ ]*) (?<path>[^"]*)"/.match(item[:message])
           unless detail.nil?
-            record[:method] = detail[:method]
-            record[:path] = detail[:path]
+            record['method'] = detail[:method]
+            record['path'] = detail[:path]
           end
 
           detail = /referrer: "(?<referrer>[^"]*)"/.match(item[:message])
           unless detail.nil?
-            record[:referrer] = detail[:referrer]
+            record['referrer'] = detail[:referrer]
           end
 
           return record
         end
 
         # other log
-        record[:message] = log.gsub(/\n$/, '')
+        record['message'] = log.gsub(/\n$/, '')
         record
       end
 
@@ -163,9 +212,9 @@ module Fluent
         log = record['log']
 
         record['environment'] = Socket.gethostname.split('-')[0]
-        record[:cid] = file[0]
-        record[:cname] = tag
-        record[:ctime] = record['time']
+        record['cid'] = file[0]
+        record['cname'] = tag
+        record['ctime'] = record['time']
 
         record.delete('log')
         record.delete('file')
@@ -173,26 +222,29 @@ module Fluent
 
         if /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4} [A-Z]/.match(log)
 
-          item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
+          item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) protocol:op_query (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
 
-          record[:time] = item[:time]
-          record[:severity] = item[:severity]
-          record[:component] = item[:component]
-          record[:context] = item[:context]
+          record['time'] = item[:time]
+          record['severity'] = item[:severity]
+          record['component'] = item[:component]
+          record['context'] = item[:context]
 
           if item[:message]
-            record[:message] = item[:message]
+            record['message'] = item[:message]
             return record
           end
 
           if item[:querytime]
-            record[:querytime] = item[:querytime]
+            record['querytime'] = format_str(item[:querytime])
           end
 
           if item[:query]
+            record['command'] = item[:query]
             query = /^command\s* (?<collection>[^ ]*) (?<command>.*)$/.match(item[:query])
-            record[:collection] = query[:collection]
-            record[:command] = query[:command]
+            unless query.nil?
+              record['collection'] = query[:collection]
+              record['command'] = query[:command]
+            end
           end
           
           return record
@@ -201,6 +253,91 @@ module Fluent
         record
       end
 
+      # 确认是否发送通知
+      def notice(tag, record)
+
+        unless @notice
+          return record
+        end
+
+        if tag == 'app'
+          if @app_stream && record['stream']
+            send(record) if record['stream'] == @app_stream
+          end
+
+          if @app_message.length > 0 && record['message']
+            @app_message.each do |pattern|
+              if pattern.match(record['message'])
+                send(record)
+                break
+              end
+            end
+          end
+
+          if @app_status.length > 0 && record['status']
+            send(record) if @app_status.include?(record['status'])
+          end
+
+          if @app_elapsed > 0 && record['elapsed']
+            send(record) if record['elapsed'].to_f > @app_elapsed
+          end
+
+          return record
+        end
+
+        if tag == 'lb'
+          if @lb_stream && record['stream']
+            send(record) if record['stream'] == @lb_stream
+          end
+
+          if @lb_code.length > 0 && record['code']
+            send(record) if @lb_code.include?(record['code'])
+          end
+
+          if @lb_elapsed > 0 && record['elapsed']
+            send(record) if record['elapsed'].to_f > @lb_elapsed
+          end
+    
+          return record
+        end
+
+        if tag == 'mongo'
+          if @mongo_severity.length > 0 && record['severity']
+            send(record) if @mongo_severity.include?(record['severity'])
+          end
+
+          if @mongo_querytime > 0 && record['querytime']
+            send(record) if record['querytime'].to_f > @mongo_querytime
+          end
+
+          return record
+        end
+
+        record
+        
+      end
+
+      # 发送UDP请求
+      def send(record)
+
+        log.info('send udp notice', record)
+
+        data = {
+          :query => '/api/log/notice',
+          :params => {
+            :data => record
+          }
+        }
+        @udp.send(data.to_json, 0, Socket.pack_sockaddr_in(@port, @host))
+      end
+
+      # 转数字
+      def format_str(str)
+        return str.to_i if (str =~ /^\d+$/)
+        return str.to_f if (str =~ /^\d+\.\d+$/)
+        str
+      end
+
     end
   end
 end

data/sample/README.md

@@ -21,12 +21,14 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
 	% gem signin
 	
 - 发布
-	rake release
+  1. 删除pkg目录下的所有内容
+  2. 修改 fluent-plugin-light-core.gemspec 中的 spec.version
+  3. 提交所有的修改
+	4. 执行打包发布 $ rake release
 
 
 ## 调试
 - 执行fluentd
 
-  % cd sample
-  % fluentd -c source.conf -p ../lib/fluent/plugin
-
+  % rm -f sample/*.pos
+  % fluentd -c sample/source.conf -p lib/fluent/plugin

data/sample/udpsample.rb

@@ -0,0 +1,20 @@
+
+require 'socket'
+require 'json'
+
+udp = UDPSocket.open()
+udp.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, 1)
+udp_addr = Socket.pack_sockaddr_in(7000, '255.255.255.255')
+
+data = {
+  :query => '/api/log/notice',
+  :params => {
+    :data => {:A => 'A'}
+  }
+}
+
+params = data.to_json
+params = "{\"query\":\"/api/log/notice\",\"params\":{\"data\":{\"stream\":\"stdout\",\"environment\":\"Mac\",\"cid\":\"app-0c1b728705d8-f5fc78bc8-8rjfl\",\"cname\":\"0c1b728705d8\",\"ctime\":\"2020-09-20T06:00:53.516987008Z\",\"time\":\"2020-09-20T06:00:53.516\",\"component\":\"I\",\"method\":\"GET\",\"url\":\"/api/app/last?id=xundianxb&os=android\",\"status\":\"200\",\"size\":\"349\",\"uid\":\"-\",\"elapsed\":\"76.078\",\"addr\":\"::ffff:10.244.4.86\"}}}"
+udp.send(params, 0, udp_addr)
+
+udp.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-light-core
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.6
 platform: ruby
 authors:
 - LIN LI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2021-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -89,6 +89,7 @@ files:
 - lib/fluent/plugin/filter_light_core.rb
 - sample/README.md
 - sample/source.conf
+- sample/udpsample.rb
 - test/helper.rb
 - test/plugin/test_filter_light_core.rb
 homepage: https://git.alphabets.cn/light/fluent-plugin-light-core