fluent-plugin-light-core 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f1224d40e14381a1b11835fcad72d2ae6ef35f8f926c002f4ec08c821090581
-  data.tar.gz: b9141a0a06ea58524df035041c08495434de34aff4991139e3d4f09659270f9d
+  metadata.gz: 262f7c2b69259e409b93285c4138386a24d7ee2d12d81f120ef1306875d0d701
+  data.tar.gz: 19ecb1abd1e754ffe2da66f72c8cb211fe81aa5a4e3543c742f63d6ee121a474
 SHA512:
-  metadata.gz: e4ffc86c4bfa5547ff4100da8f85064b4822d0d80e5e60a052366b13f7c7d92d4dd3d9beba7045d47bf3e2409b1f81cf7fa01c1ae048996d8cb19eb8af0690de
-  data.tar.gz: 652e52d0a38c28c26b4f3e7305a59d8301d53365cfeef0384fcc2abd59b78a6c4073dffb489d37329489e63fa7bb2b4569d45c7e7262d83069b402287c7d4785
+  metadata.gz: a39111482eb7fcd051f14f92d4a6f062df5f9673f0a474c2c26ea5c73ba25074a2e13237ae3dc9c67e9ec2743009c4cc59a63476dd20a98883151fb691e62090
+  data.tar.gz: de9645e74a662f9b5be93e4bdfb7141f8e473c27dff352755f7dca683ced2d505a2a7188270ef5a0a540e1a15f1bce8f5fcb15eaa963efeca89994397eb56abb

data/fluent-plugin-light-core.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-light-core"
-  spec.version = "0.1.2"
+  spec.version = "0.1.3"
   spec.authors = ["LIN LI"]
   spec.email   = ["l.li@alphabets.cn"]
 

data/lib/fluent/plugin/filter_light_core.rb

@@ -13,25 +13,73 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "fluent/plugin/filter"
+require 'fluent/plugin/filter'
+require 'socket'
+require 'json'
 
 module Fluent
   module Plugin
     class LightCoreFilter < Fluent::Plugin::Filter
-      Fluent::Plugin.register_filter("light_core", self)
+      Fluent::Plugin.register_filter('light_core', self)
+
+      # 通知相关配置
+      config_param :notice, :bool,    default: false
+      config_param :port,   :integer, default: 7000
+      config_param :host,   :string,  default: '255.255.255.255'
+
+      # Application 异常 settings 
+      config_param :app_stream,  :string, default: 'stderr'
+      config_param :app_message, :array,  default: [],      value_type: :regexp
+      config_param :app_status,  :array,  default: ['500'], value_type: :string
+      config_param :app_elapsed, :float,  default: 2000
+
+      # Nginx 异常 settings 
+      config_param :lb_stream,  :string, default: 'stderr'
+      config_param :lb_code,    :array,  default: ['500'], value_type: :string
+      config_param :lb_elapsed, :float,  default: 3000
+
+      # MongoDB 异常 settings 
+      config_param :mongo_severity,  :array, default: ['F', 'E'], value_type: :string
+      config_param :mongo_querytime, :float, default: 100
+
+      # 初始化UDP实例
+      def start
+        super
+
+        if @notice
+          log.info('init udp connection')
+          @udp = UDPSocket.open()
+          @udp.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, 1)
+        end
+      end
+
+      # 清理UDP连接
+      def shutdown
+
+        if @notice
+          log.info('close udp connection')
+          @udp.close if @udp
+        end
+
+        super
+      end
 
+      # 主处理
       def filter(tag, time, record)
 
         if tag == 'app'
-          return filter_app(tag, time, record)
+          record = filter_app(tag, time, record)
+          return notice('app', record)
         end
 
         if ['lb', 'hub'].include? tag
-          return filter_lb(tag, time, record)
+          record = filter_lb(tag, time, record)
+          return notice('lb', record)
         end
 
         if ['mongo', 'secondary', 'arbiter'].include? tag
-          return filter_mongo(tag, time, record)
+          record = filter_mongo(tag, time, record)
+          return notice('mongo', record)
         end
 
         record
@@ -109,7 +157,7 @@ module Fluent
         
         # access log
         if /^[^ ]+ [^ ]+ [^ ]+ \[\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+\d{2}:\d{2}\]/.match(log)
-          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)")?/.match(log)
+          item = /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" "(?<forwarder>[^\"]*)" "(?<elapsed>[^\"]*)")?/.match(log)
 
           record['remote'] = item[:remote]
           record['host'] = item[:host]
@@ -122,6 +170,7 @@ module Fluent
           record['referer'] = item[:referer]
           record['agent'] = item[:agent]
           record['forwarder'] = item[:forwarder]
+          record['elapsed'] = item[:elapsed]
 
           return record
         end
@@ -173,7 +222,7 @@ module Fluent
 
         if /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4} [A-Z]/.match(log)
 
-          item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
+          item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) protocol:op_query (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
 
           record['time'] = item[:time]
           record['severity'] = item[:severity]
@@ -201,6 +250,84 @@ module Fluent
         record
       end
 
+      # 确认是否发送通知
+      def notice(tag, record)
+
+        unless @notice
+          return record
+        end
+
+        if tag == 'app'
+          if @app_stream && record['stream']
+            send(record) if record['stream'] == @app_stream
+          end
+
+          if @app_message.length > 0 && record['message']
+            @app_message.each do |pattern|
+              if pattern.match(record['message'])
+                send(record)
+                break
+              end
+            end
+          end
+
+          if @app_status.length > 0 && record['status']
+            send(record) if @app_status.include?(record['status'])
+          end
+
+          if @app_elapsed > 0 && record['elapsed']
+            send(record) if record['elapsed'].to_f > @app_elapsed
+          end
+
+          return record
+        end
+
+        if tag == 'lb'
+          if @lb_stream && record['stream']
+            send(record) if record['stream'] == @lb_stream
+          end
+
+          if @lb_code.length > 0 && record['code']
+            send(record) if @lb_code.include?(record['code'])
+          end
+
+          if @lb_elapsed > 0 && record['elapsed']
+            send(record) if record['elapsed'].to_f > @lb_elapsed
+          end
+    
+          return record
+        end
+
+        if tag == 'mongo'
+          if @mongo_severity.length > 0 && record['severity']
+            send(record) if @mongo_severity.include?(record['severity'])
+          end
+
+          if @mongo_querytime > 0 && record['querytime']
+            send(record) if record['querytime'].to_f > @mongo_querytime
+          end
+
+          return record
+        end
+
+        record
+        
+      end
+
+      # 发送UDP请求
+      def send(record)
+
+        log.info('send udp notice', record)
+
+        data = {
+          :query => '/api/log/notice',
+          :params => {
+            :data => record
+          }
+        }
+        @udp.send(data.to_json, 0, Socket.pack_sockaddr_in(@port, @host))
+      end
+
     end
   end
 end

data/sample/README.md

@@ -27,4 +27,5 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
 ## 调试
 - 执行fluentd
 
+  % rm -f sample/*.pos
   % fluentd -c sample/source.conf -p lib/fluent/plugin

data/sample/udpsample.rb

@@ -0,0 +1,20 @@
+
+require 'socket'
+require 'json'
+
+udp = UDPSocket.open()
+udp.setsockopt(Socket::SOL_SOCKET, Socket::SO_BROADCAST, 1)
+udp_addr = Socket.pack_sockaddr_in(7000, '255.255.255.255')
+
+data = {
+  :query => '/api/log/notice',
+  :params => {
+    :data => {:A => 'A'}
+  }
+}
+
+params = data.to_json
+params = "{\"query\":\"/api/log/notice\",\"params\":{\"data\":{\"stream\":\"stdout\",\"environment\":\"Mac\",\"cid\":\"app-0c1b728705d8-f5fc78bc8-8rjfl\",\"cname\":\"0c1b728705d8\",\"ctime\":\"2020-09-20T06:00:53.516987008Z\",\"time\":\"2020-09-20T06:00:53.516\",\"component\":\"I\",\"method\":\"GET\",\"url\":\"/api/app/last?id=xundianxb&os=android\",\"status\":\"200\",\"size\":\"349\",\"uid\":\"-\",\"elapsed\":\"76.078\",\"addr\":\"::ffff:10.244.4.86\"}}}"
+udp.send(params, 0, udp_addr)
+
+udp.close

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-light-core
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - LIN LI
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-23 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -89,6 +89,7 @@ files:
 - lib/fluent/plugin/filter_light_core.rb
 - sample/README.md
 - sample/source.conf
+- sample/udpsample.rb
 - test/helper.rb
 - test/plugin/test_filter_light_core.rb
 homepage: https://git.alphabets.cn/light/fluent-plugin-light-core