fluent-plugin-light-core 0.2.1 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af0f4865440862be4488030659ec9798b8eb9261bc2f9b95ee093a0b5de28387
-  data.tar.gz: 5a46ab336bb472b47f6572d5836e6fbbfeea8075e19dfa6a6781d4e938ceddb7
+  metadata.gz: 840e2f50ffd4383b36db2e72443db1161a9252bf04e4eb305d721af1ce31bb64
+  data.tar.gz: b9bf12844a61596b32252fa36266a1a9e875f8314d087993d94874993a698ad3
 SHA512:
-  metadata.gz: 045f2e53eba3fcf2ac10aa1a625e539bff38f6221c4a776d4c7a8b5250868e8bda1b0c1c59b07da1458c86787f07e6f9d568e4cb73562b7a4d261aaa0a42be1b
-  data.tar.gz: 7272c0c901fd28c3aec6b9241060f8dab6e64f3cc91f40ab7a044f866b951e4b3aa4cc4ac9428029914e7104ca43840abceee5b88cae3e0d0a95928ed337311c
+  metadata.gz: d67d38209e5ad74bf70a9ff7b4cb9949072c9009632616ad8ba88cb50e49fd2f3dd207e7e5eda1e304d416c3dd3ecfaa88567568ccfdda477f7e2da5491e7e27
+  data.tar.gz: f6ed9b675e53d81e54c603972a3e8ba75b6218626fd17f8cb21ab75a89f4644d6e8973d046e6aec4f776cc5c3b1221b660faf797906be01d136efb88d35ea48d

data/.gitignore

@@ -1,2 +1,3 @@
 *.log
 *.pos
+*.gem

data/Gemfile

@@ -2,4 +2,6 @@ source "https://rubygems.org"
 
 gemspec
 
-gem "sentry-ruby"
+gem "sentry-ruby"
+
+gem "oj", "~> 3.13"

data/Gemfile.lock

@@ -1,38 +1,71 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-light-core (0.1.0)
-      fluentd (>= 0.14.10, < 2)
+    fluent-plugin-light-core (0.2.3)
+      fluentd (>= 1.14.2, < 2)
+      sentry-ruby (>= 4.8.0, < 5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    cool.io (1.6.0)
-    fluentd (1.11.2)
+    concurrent-ruby (1.1.9)
+    cool.io (1.7.1)
+    faraday (1.8.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
+      faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
+      faraday-rack (~> 1.0)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
+    faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.2.0)
+    faraday-patron (1.0.0)
+    faraday-rack (1.0.0)
+    fluentd (1.14.2)
+      bundler
       cool.io (>= 1.4.5, < 2.0.0)
-      http_parser.rb (>= 0.5.1, < 0.7.0)
+      http_parser.rb (>= 0.5.1, < 0.8.0)
       msgpack (>= 1.3.1, < 2.0.0)
-      serverengine (>= 2.0.4, < 3.0.0)
+      serverengine (>= 2.2.2, < 3.0.0)
       sigdump (~> 0.2.2)
-      strptime (>= 0.2.2, < 1.0.0)
+      strptime (>= 0.2.4, < 1.0.0)
       tzinfo (>= 1.0, < 3.0)
       tzinfo-data (~> 1.0)
+      webrick (>= 1.4.2, < 1.8.0)
       yajl-ruby (~> 1.0)
-    http_parser.rb (0.6.0)
-    msgpack (1.3.3)
+    http_parser.rb (0.7.0)
+    msgpack (1.4.2)
+    multipart-post (2.1.1)
+    oj (3.13.9)
     power_assert (1.1.3)
     rake (12.3.2)
-    serverengine (2.2.1)
+    ruby2_keywords (0.0.5)
+    sentry-ruby (4.8.0)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      faraday (>= 1.0)
+      sentry-ruby-core (= 4.8.0)
+    sentry-ruby-core (4.8.0)
+      concurrent-ruby
+      faraday
+    serverengine (2.2.4)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
-    strptime (0.2.4)
+    strptime (0.2.5)
     test-unit (3.2.9)
       power_assert
-    thread_safe (0.3.6)
-    tzinfo (1.2.5)
-      thread_safe (~> 0.1)
-    tzinfo-data (1.2020.1)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    tzinfo-data (1.2021.5)
       tzinfo (>= 1.0.0)
+    webrick (1.7.0)
     yajl-ruby (1.4.1)
 
 PLATFORMS
@@ -41,7 +74,9 @@ PLATFORMS
 DEPENDENCIES
   bundler (~> 1.14)
   fluent-plugin-light-core!
+  oj (~> 3.13)
   rake (~> 12.0)
+  sentry-ruby
   test-unit (~> 3.0)
 
 BUNDLED WITH

data/fluent-plugin-light-core.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-light-core"
-  spec.version = "0.2.1"
+  spec.version = "0.2.5"
   spec.authors = ["LIN LI"]
   spec.email   = ["l.li@alphabets.cn"]
 
@@ -23,6 +23,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.14"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
-  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
-  spec.add_runtime_dependency "sentry-ruby", [">= 4.1.5", "< 5"]
+  spec.add_runtime_dependency "fluentd", [">= 1.14.2", "< 2"]
+  spec.add_runtime_dependency "sentry-ruby", [">= 4.8.0", "< 5"]
+  spec.add_runtime_dependency "oj", [">= 3.13", "< 4"]
 end

data/lib/fluent/plugin/filter_light_core.rb

@@ -36,6 +36,7 @@ module Fluent
       config_param :lb_stream,  :string, default: 'stderr'
       config_param :lb_code,    :array,  default: ['400', '500'], value_type: :string
       config_param :lb_elapsed, :float,  default: 3
+      config_param :lb_ignore,  :array,  default: [], value_type: :string
 
       # MongoDB 异常 settings 
       config_param :mongo_severity,  :array, default: ['F', 'E'], value_type: :string
@@ -130,9 +131,9 @@ module Fluent
           record['method'] = item[:method]
           record['url'] = item[:url]
           record['status'] = item[:status]
-          record['size'] = format_str(item[:size])
+          record['size'] = str_to_num(item[:size])
           record['uid'] = item[:uid]
-          record['elapsed'] = format_str(item[:elapsed])
+          record['elapsed'] = str_to_num(item[:elapsed])
           record['addr'] = item[:addr].gsub(/\n$/, '')
 
           return record
@@ -169,10 +170,10 @@ module Fluent
           record['method'] = item[:method]
           record['path'] = item[:path]
           record['code'] = item[:code]
-          record['size'] = format_str(item[:size])
+          record['size'] = str_to_num(item[:size])
           record['referer'] = item[:referer]
           record['agent'] = item[:agent]
-          record['elapsed'] = format_str(item[:elapsed])
+          record['elapsed'] = str_to_num(item[:elapsed])
           record['requestid'] = item[:requestid]
           record['sessionid'] = item[:sessionid]
 
@@ -183,7 +184,7 @@ module Fluent
         if /^\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2} \[error\]/.match(log)
           item = /^(?<time>[^ ]+ [^ ]+) \[(?<level>.*)\] (?<pid>\d*)#(?<tid>[^:]*): \*(?<cid>\d*) (?<message>.*)$/.match(log)
 
-          record['time'] = item[:time]
+          record['time'] = record['ctime']
           record['level'] = item[:level]
           record['message'] = item[:message]
           record['process'] = item[:pid]     # process id
@@ -224,61 +225,26 @@ module Fluent
         record.delete('file')
         record.delete('time')
 
-        # 旧版本日志格式解析
-        if /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}\+\d{4} [A-Z]/.match(log)
-
-          item = /^(?<time>[^ ]*) (?<severity>[A-Z])\s* (?<component>(-|([^ ]*)))\s* \[(?<context>[^\]]*)\]\s* ((?<query>.*) protocol:op_query (?<querytime>[\d\.]+(?=ms))|(?<message>.*))/.match(log)
-
-          record['time'] = item[:time]
-          record['severity'] = item[:severity]
-          record['component'] = item[:component]
-          record['context'] = item[:context]
-
-          if item[:message]
-            record['message'] = item[:message]
-            return record
-          end
-
-          if item[:querytime]
-            record['querytime'] = format_str(item[:querytime])
-          end
-
-          if item[:query]
-            record['command'] = item[:query]
-            query = /^command\s* (?<collection>[^ ]*) (?<command>.*)$/.match(item[:query])
-            unless query.nil?
-              record['collection'] = query[:collection]
-              record['command'] = query[:command]
-            end
-          end
-          
-          return record
-        else
-
-          # 版本4.4开始，默认日志为json格式
-          item = JSON.parse(log)
-          record['time'] = item['t']['$date']
-          record['severity'] = item['s']
-          record['component'] = item['c']
-          record['context'] = item['ctx']
-          record['identifier'] = item['id']
-          record['message'] = item['msg']
-
-          attributes = item['attr']
-          if attributes
-            record['querytime'] = attributes['durationMillis']
-            record['collection'] = attributes['ns']
-            record['command'] = attributes['command']
-            record['attr'] = attributes
-          end
-
-          record['tags'] = item['tags']
-          record['truncated'] = item['truncated']
-          record['size'] = item['size']
-        
-          return record
+        # 版本4.4开始，默认日志为json格式
+        item = JSON.parse(log)
+        record['time'] = item['t']['$date']
+        record['severity'] = item['s']
+        record['component'] = item['c']
+        record['context'] = item['ctx']
+        record['identifier'] = item['id']
+        record['message'] = item['msg']
+
+        attributes = item['attr']
+        if attributes
+          record['querytime'] = attributes['durationMillis']
+          record['collection'] = attributes['ns']
+          record['command'] = attributes['command']
+          record['attr'] = attributes
         end
 
+        record['tags'] = item['tags']
+        record['truncated'] = item['truncated']
+        record['size'] = item['size']
         record
       end
 
@@ -291,16 +257,15 @@ module Fluent
 
         if tag == 'app'
 
-          # 未使用
           if @app_stream && record['stream']
-            send(record) if record['stream'] == @app_stream
+            message = 'Stream error' + record['stream']
+            send(tag, message, record) if record['stream'] == @app_stream
           end
 
-          # 未使用
           if @app_message.length > 0 && record['message']
             @app_message.each do |pattern|
               if pattern.match(record['message'])
-                send(record)
+                send(tag, record['message'], record)
                 break
               end
             end
@@ -320,6 +285,11 @@ module Fluent
         end
 
         if tag == 'lb'
+
+          if @lb_ignore && record['path'] && @lb_ignore.include?(record['path'])
+            return record
+          end
+
           if @lb_stream && record['stream']
             send(tag, 'Stderror', record) if record['stream'] == @lb_stream
           end
@@ -359,11 +329,13 @@ module Fluent
       # 发送UDP请求
       def send(tag, message, record)
         log.debug('send udp notice')
-        Sentry.capture_message(message, :extra => record, :tags => {'log' => tag})
+        encoded = message.force_encoding(Encoding::UTF_8)
+        Sentry.capture_message(encoded, :extra => record, :tags => {'log' => tag}) rescue log.warn(encoded)
       end
 
       # 转数字
-      def format_str(str)
+      def str_to_num(str)
+        return 0 if (str == '-')
         return str.to_i if (str =~ /^\d+$/)
         return str.to_f if (str =~ /^\d+\.\d+$/)
         str

data/sample/README.md

@@ -42,3 +42,8 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
 - 安装依赖
 
   gem install sentry-ruby
+
+- 更新依赖版本
+
+  bundle install
+  会修改Gemfile.lock文件

data/sample/source.conf

@@ -86,6 +86,7 @@
   lb_stream        stderr
   lb_code          400,500
   lb_elapsed       5
+  lb_ignore        /api/environment/list,/api/certificate/list?condition%5Bkind%5D=SSL
   app_status       500
   app_elapsed      5
 </filter>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-light-core
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.5
 platform: ruby
 authors:
 - LIN LI
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.14.2
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +68,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.14.2
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -78,7 +78,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.5
+        version: 4.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '5'
@@ -88,10 +88,30 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.1.5
+        version: 4.8.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '5'
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.13'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
 description: light core fluent plugin. support mongodb, nginx and application
 email:
 - l.li@alphabets.cn
@@ -107,7 +127,6 @@ files:
 - Rakefile
 - fluent-plugin-light-core.gemspec
 - lib/fluent/plugin/filter_light_core.rb
-- pkg/fluent-plugin-light-core-0.2.0.gem
 - sample/README.md
 - sample/source.conf
 - sample/udpsample.rb
@@ -117,7 +136,7 @@ homepage: https://git.alphabets.cn/light/fluent-plugin-light-core
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -133,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: light core fluent plugin.
 test_files: