fluent-plugin-light-core 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 153a8ac1fcd1f93b50ca3502cac1d2a941f283c707e50b6367f42b8c13e08cf7
-  data.tar.gz: a27cd131cdec3ecfeddb80d4d154809a6745ca652a229ec6970dad135cf57e1b
+  metadata.gz: 0c276f56914554fa43af82f0ce4a8c8e391f77b5f9e8c7d3c645b60b45c6ba77
+  data.tar.gz: c039e954174ad34042adf4ed68c9bffbbee2ff99f955deb33ac9cd613e46b10a
 SHA512:
-  metadata.gz: 44197cea697e733b50abad6bde8e7e645a2d3c4fee3161de2b131461dadb8bc5076581ec9905975c173bddc5b726ff45bf08dcb9442d7784e208c1c7b5af0f80
-  data.tar.gz: 961ad750fe2f00b933792e8278822e927942763a64db38515159e44b3ce4145c93d9801a5cf36fb73a9e3b6e07fdc195bbda69b2d3a895c1314860ae1d286363
+  metadata.gz: 903b498c54d3ede06a6360211f2eeeff2da5e9d1ed2d1d522136b78b75e39368c9a41e4d6340f6b8565cb0fba364b10f6f43fc0a7e8d74432ce03dfc5439d48a
+  data.tar.gz: 50b9f395eeafaa166daa8f95c0b9261d9397a6d61608b42326b2c08b5b2e0b72986a9879978936a87acc2238c4e9e688e577533da5a48883a5a431d266735ba1

data/.gitignore

@@ -1,3 +1,5 @@
 *.log
 *.pos
 *.gem
+
+audit.json

data/fluent-plugin-light-core.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-light-core"
-  spec.version = "0.3.3"
+  spec.version = "0.3.4"
   spec.authors = ["LIN LI"]
   spec.email   = ["l.li@alphabets.cn"]
 

data/lib/fluent/plugin/filter_light_core.rb

@@ -76,23 +76,27 @@ module Fluent
         # 应用
         if ['app', 'service'].include? tag
           record = filter_app(tag, time, record)
+          return unless record
           notice('app', record)
         end
 
         # 负载均衡
         if ['lb'].include? tag
           record = filter_lb(tag, time, record)
+          return unless record
           notice('lb', record)
         end
 
-        # 数据库 - TODO: 其中mongo为旧tag删除预定
-        if ['mongo', 'master', 'secondary', 'arbiter'].include? tag
+        # 数据库
+        if ['master', 'secondary', 'arbiter'].include? tag
           record = filter_mongo(tag, time, record)
+          return unless record
           notice('mongo', record)
         end
 
         if ['syslog.messages', 'syslog.secure', 'syslog.audit'].include? tag
           record = filter_syslog(tag, time, record)
+          return unless record
         end
 
         record['environment'] = ENV['FLUENTD_ENV']
@@ -241,7 +245,7 @@ module Fluent
       def filter_mongo(tag, time, record)
 
         file = record['file'].split('/').last.split('_')
-        log = record['log']
+        logging = record['log']
 
         record['cid'] = file[0]
         record['cname'] = tag
@@ -252,7 +256,13 @@ module Fluent
         record.delete('time')
 
         # 版本4.4开始，默认日志为json格式
-        item = JSON.parse(log)
+        begin
+          item = JSON.parse(logging)
+        rescue JSON::ParserError
+          log.warn('json parse error : ', logging)
+          return nil
+        end
+
         record['time'] = item['t']['$date']
         record['severity'] = item['s']
         record['component'] = item['c']
@@ -264,7 +274,6 @@ module Fluent
         if attributes
           record['querytime'] = attributes['durationMillis']
           record['collection'] = attributes['ns']
-          record['command'] = attributes['command']
           record['reslen'] = attributes['reslen']
           storage = attributes['storage']
           if storage

data/sample/README.md

@@ -52,14 +52,30 @@ Initialized empty Git repository in /Users/lilin/developer/light/fluent-plugin-l
 ## 在Linxu上运行
 
 - 进入fluentd容器, 安装依赖
+```
 $ yum groupinstall 'Development Tools'
 $ yum install rubygems ruby-devel
+```
 
 - 下载代码, 安装依赖
+```
 $ git clone http://git.alphabets.cn/light/fluent-plugin-light-core.git
 $ cd fluent-plugin-light-core
 $ gem install bundler:1.17.2
 $ bundler install
+```
 
 - 运行代码
+```
 $ rm -f sample/*.pos && fluentd -c sample/source.conf -p lib/fluent/plugin
+```
+
+## MySQL数据提取
+```
+sudo gem install fluent-plugin-sql --no-document
+sudo gem install mysql2 --no-document
+  mac下mysql2无法安装 报错
+  ld: library not found for -lssl
+  可以用下面的命令
+  sudo gem install mysql2 -- --with-opt-dir="$(brew --prefix openssl)"
+```

data/sample/source.conf

@@ -38,18 +38,18 @@
 #   time_format      %Y-%m-%dT%H:%M:%S.%NZ
 # </source>
 
-# <source>
-#   @type            tail
-#   path             sample/db*.log
-#   pos_file         sample/source.mongo.pos
-#   tag              mongo
-#   format           json
-#   read_from_head   true
-#   path_key         file
-#   time_key         time
-#   keep_time_key    true
-#   time_format      %Y-%m-%dT%H:%M:%S.%NZ
-# </source>
+<source>
+  @type            tail
+  path             sample/db*.log
+  pos_file         sample/source.master.pos
+  tag              master
+  format           json
+  read_from_head   true
+  path_key         file
+  time_key         time
+  keep_time_key    true
+  time_format      %Y-%m-%dT%H:%M:%S.%NZ
+</source>
 
 # <source>
 #   @type            tail
@@ -115,6 +115,58 @@
 #   </parse>
 # </source>
 
+# <source>
+#   @type          tail
+#   path           sample/mongo/master/audit.json,sample/mongo/secondary/audit.json
+#   pos_file       sample/mongo/fluentd-mongodb-audit.pos
+#   read_from_head true
+#   tag            mongodb.audit
+#   format         json
+#   path_key       file
+# </source>
+
+#######################################
+# mysql
+#######################################
+# <source>
+#   @type           sql
+#   host            mysql
+#   port            53306
+#   database        jumpserver
+#   adapter         mysql2
+#   username        jumpserver
+#   password        yqk1EkO3eB(j50FR5anXSNT@oXfMil
+#   tag_prefix      fortress
+#   select_interval 60s  # optional
+#   select_limit    500  # optional
+#   state_file      /var/log/fluentd-sql-state.pos
+
+#   <table>
+#     table         audits_operatelog
+#     tag           operatelog
+#     update_column datetime
+#   </table>
+
+#   <table>
+#     table         audits_passwordchangelog
+#     tag           passwordchangelog
+#     update_column datetime
+#   </table>
+
+#   <table>
+#     table         audits_userloginlog
+#     tag           userloginlog
+#     update_column datetime
+#   </table>
+
+#   <table>
+#     table         terminal_command
+#     tag           command
+#     update_column timestamp
+#   </table>
+
+# </source>
+
 <filter **>
   @type            light_core
   sentry           true
@@ -140,6 +192,28 @@
 #   </rule>
 # </match>
 
+# <match **>
+#   @type                copy
+
+#   <store>
+#     @type              elasticsearch
+#     host               es-master
+#     port               9200
+#     logstash_format    true
+#     logstash_prefix    ${tag}
+#     <buffer>
+#       @type              file
+#       path               sample/fluentd
+#       flush_interval     5s
+#       flush_thread_count 3  # 收集日志的线程数，太大回占用过多的cpu资源，太小收集日志不及时
+#     </buffer>
+#     log_es_400_reason  true
+#     reconnect_on_error true
+#     request_timeout    10s
+#     suppress_type_name true # 去除 type_name 警告
+#   </store>
+# </match>
+
 <match **>
   @type            stdout
 </match>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-light-core
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - LIN LI
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-18 00:00:00.000000000 Z
+date: 2022-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler