logstash-output-thinkingdata 0.1.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c02a307fd4272626a3066db6f4d924767fa058dbb889477583d19db2b195b9e3
-  data.tar.gz: e019db8cd60d9fd6625d6a16391f2bcd899c88082442d6004579b3213fc8e7e4
+  metadata.gz: 6859f106a23f8b6f24120c2d51905b7b09e0c9b713235319b0c71f60c105325b
+  data.tar.gz: 50fe28c3a55a3029473afbe70ba115d0e38aa7b17da1c3acacd3572b6e4915a9
 SHA512:
-  metadata.gz: f990de3bab4004e223ad6f9691ec22f217f51ed711d617036fd14527ddc73a4fa353da3aae9713a95533006cb0793aba44aa9b999d57acc9369da7e590c2dc70
-  data.tar.gz: 92787899f34fc00787ec28130b91013a8197e5ba22534a6958a0fec36c602ef5dd5a816c9f6744818cf8f9ddb941c89c2cd9b087f8efa0ed713dd39f6552d077
+  metadata.gz: dda99ed3e2c10d2c7b86e09e71b01f1074609829791d8cb1b8fd42daf54c173cb72d3f644b007b7592d3652e8068cfd2cc6179ff25f80909c1bec292a68462e1
+  data.tar.gz: 3fe885fd3bcd52ef37442d52fbbe37a6b37835d80101f0afb179ecd1b3034e50834436f45dfe6248d5d7b4b511fda4e6b6ee4bdf95d6519641008361c35ce722

data/CHANGELOG.md

@@ -1,2 +1,6 @@
+**v1.0.0** (2020-06-09)
+  - 接收Logstash中event传递的message发送到TA中
+  - filebeat 日志打印支持
+
 **v0.1.1** (2020-04-27)
   - thinkingdata output plugin created with the logstash plugin generator

data/lib/logstash/outputs/thinkingdata.rb

@@ -28,12 +28,22 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   config :flush_interval_sec, :validate => :number, :default => 2
 
   # 批次flush 最大 content 数量
-  config :flush_batch_size, :validate => :number, :default => 100
+  config :flush_batch_size, :validate => :number, :default => 500
 
   # 是否压缩数据 0:不压缩，1:gzip压缩
   config :compress, :validate => :number, :default => 1
 
-  PLUGIN_VERSION = "0.1.1"
+  #是否开启uuid
+  config :uuid, :validate => :boolean, :default => false
+
+
+  # 开启 filebeat 状态记录
+  config :is_filebeat_status_record, :validate => :boolean, :default => true
+
+  # 是否检测appid
+  config :appid_check, :validate => :boolean, :default => false
+
+  PLUGIN_VERSION = "1.0.0"
 
   public
   def register
@@ -42,24 +52,38 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
                  :appid => @appid,
                  :flush_interval_sec => @flush_interval_sec,
                  :flush_batch_size => @flush_batch_size,
-                 :compress =>@compress
+                 :compress =>@compress,
+                 :uuid => @uuid,
+                 :is_filebeat_status_record => @is_filebeat_status_record,
+                 :appid_check => @appid_check
     )
 
     http_client_config = client_config
     http_client_config[:user_agent] = "thinkingdata_logstash_output_plugin_" + PLUGIN_VERSION
     @client = Manticore::Client.new(http_client_config)
-    appid_check
+    ta_appid_check if @appid_check
+    @receive_count = 0
+    @parse_error_count = 0
+    @last_report_count = 0
+    @total_send_count = 0
     buffer_config = {
         :max_items => @flush_batch_size.to_i,
         :max_interval => @flush_interval_sec.to_i,
         :logger => @logger
     }
     buffer_initialize(buffer_config)
+    @filebeat_status = {} if @is_filebeat_status_record
+    @report_thread = Thread.new do
+      loop do
+        sleep 60
+        report
+      end
+    end
   end # def register
 
   #验证appid
-  public
-  def appid_check
+  private
+  def ta_appid_check
     @server_uri = URI(@url)
     @server_uri.path = "/check_appid"
     @server_uri.query= "appid="+@appid
@@ -74,26 +98,39 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   public
   def multi_receive(events)
     return if events.empty?
-
-    events.each do |e|
+    @receive_count += events.length
+    events.each do |event|
       begin
-        content = JSON.parse(e.get("message"))
+        content = LogStash::Json.load(event.get("message"))
+        content['#uuid'] = SecureRandom.uuid if @uuid
+        if is_filebeat_input?(event)   #filebeat input 记录
+          host = event.get("[host][name]")
+          file = event.get("[log][file][path]")
+          file = event.get("[source]")if file.nil?
+          offset = event.get("[log][offset]")
+          offset = event.get("[offset]") if offset.nil?
+          log_detail = "host: #{host}, file: #{file}"
+          record_filebeat_status(log_detail, offset) if @is_filebeat_status_record
+        end
         buffer_receive(content)
-      rescue
-        @logger.error("Could not process content", :content => e.to_s)
+      rescue => e
+        @logger.error("Could not process content", :content => event.to_s, :Exception => e)
+        @parse_error_count += 1
       end
     end
-    # return "Event received"
   end # def event
 
   public
   def close
+    buffer_state[:timer].kill
     buffer_flush(:final => true)
-    client.close
+    @report_thread.kill
+    @client.close
+    report
   end
 
   public
-  def flush(events, final = false)
+  def flush(events, final)
     if @compress == 0
        data = events.to_json
        compress_type = 'none'
@@ -108,14 +145,71 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
     end
       headers ={'appid' => @appid,'version'=>PLUGIN_VERSION,'user-agent' =>'logstash_'+ PLUGIN_VERSION,
               'compress' => compress_type}
-      response = client.post(@url, :body => data,:headers =>headers).call
-      result = JSON.parse(response.body)
-    if response.code != 200
-        @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
-        raise
+     until do_send(data,headers)
+       sleep 5
+     end
+    @total_send_count += events.length
+  end
+
+  private
+  def do_send(data, headers)
+    begin
+      response = @client.post(@url, :body => data,:headers =>headers).call
+      if response.code != 200
+        @logger.error("Send failed, code: #{response.code}, body: #{response.body}",:url => @url)
+         return false
+      end
+    rescue => e
+      @logger.error("Send failed",:url => @url, :exception => e.class.name, :backtrace => e.backtrace)
+      return false
     end
-    if result['code'] != 0
-      @logger.error(" Send failed data:" , :failData => events.to_json,:responseBody => response.body)
+    true
+  end
+
+  private
+  def is_filebeat_input?(event)
+    type = event.get("[agent][type]")
+    return true if !type.nil? && type == "filebeat"
+    type = event.get("[@metadata][beat]")
+    return true if !type.nil? && type == "filebeat"
+    type = event.get("[beat]")
+    return true unless type.nil?
+    false
+  end
+
+  private
+  def record_filebeat_status(log_detail, offset)
+    status = @filebeat_status[log_detail]
+    if status.nil?
+      status = {:receive_time => Time.now, :offset => offset}
+      @filebeat_status[log_detail] = status
+    else
+      status[:offset] = offset
+      status[:receive_time] = Time.now
     end
   end
+
+  private
+  def report
+     interval_count = @total_send_count - @last_report_count
+     @last_report_count = @total_send_count
+     @logger.info("Report",
+                  "IntervalReceive(records):" => interval_count,
+                  "Receive(records):" => @receive_count,
+                  "TotalSend(records)" => @total_send_count,
+                  "ParseError(records)" => @parse_error_count)
+     @logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status !={}
+  end
+
+  private
+
+  def format_filebeat_report
+    result = "\n"
+    @filebeat_status.each do |k, v|
+      result << k << "=>" << v.to_s << "\n"
+    end
+    @filebeat_status = {}
+    result
+  end
+
 end # class LogStash::Outputs::Thinkingdata

data/logstash-output-thinkingdata.gemspec

@@ -1,12 +1,12 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-thinkingdata'
-  s.version       = '0.1.1'
+  s.version       = '1.0.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Output plugin for Thinkingdata Analytics'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
   s.homepage      = 'https://www.thinkingdata.cn/manual/api.html'
-  s.authors       = ['xuzz']
-  s.email         = 'xuzz@thinkingdata.cn'
+  s.authors       = ['sdk']
+  s.email         = 'sdk@thinkingdata.cn'
   s.require_paths = ['lib']
 
   # Files

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-thinkingdata
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 1.0.0
 platform: ruby
 authors:
-- xuzz
+- sdk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-27 00:00:00.000000000 Z
+date: 2020-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -123,7 +123,7 @@ dependencies:
 description: This gem is a logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program.
-email: xuzz@thinkingdata.cn
+email: sdk@thinkingdata.cn
 executables: []
 extensions: []
 extra_rdoc_files: []