logstash-output-sensors_analytics 0.1.2 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04a13a5a7e38d72dba2dc7064a3c567789c7109dc06cc2a7fc5b4fb9d0d0a1eb
-  data.tar.gz: 46c542e7c46e32f22d30e5d503b3163b29bea49b18ee5161d1d4a28d027e5357
+  metadata.gz: bbcf496f3c8862f0febbb3fbb018ee6a4275493437f9d7dd4137794a6e9cd214
+  data.tar.gz: fe931cc0014ae67b9bf6e11f0de3a4a8ed6dd2b06c24567f84edc09c11ad8ab3
 SHA512:
-  metadata.gz: 6c0bdcb072ceb310f92b25e5da47d89d563c56e5d84fcb99229a6e0136a8d3f76cde19062187c1d965ba61d866ab7518273252e33d98547585eb8529c1998c4e
-  data.tar.gz: d97909fd5156adc44cdfd2eb35be5479934a9019311095bdcc10378aa7435f39477e51e2d28cea8a10891dc9edd90ed97461d952eff426a559d1d635ce47403a
+  metadata.gz: cc258f6d684517793e9fd76f10000ee6c3cf74f6c89dcbe72974eff67e0a4b3956e55906aab5435fab87e1ec02ad10ab432199c7915e50883ddc00d056acb2b6
+  data.tar.gz: e86c0cc001ed9c7dba6820b81d99036e672daf2add49ef8dcfc04905931ba212c86b91067110beca6556537e91d4d28815860e4621df6c288533a9b0bc9f3a64

data/lib/logstash/outputs/sensors_analytics.rb

@@ -32,7 +32,7 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
   # 开启 filebeat 状态记录
   config :enable_filebeat_status_report, :validate => :boolean, :default => true
 
-  PLUGIN_VERSION = "0.1.2"
+  PLUGIN_VERSION = "0.1.4"
 
   public
 
@@ -116,8 +116,17 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
 
         buffer_item = @buffer_items[buffer_index(tag)]
         buffer_item.buffer_receive(record)
-      rescue
-        @logger.error("Could not process record", :record => e.to_s)
+      rescue => ex
+
+        @logger.error("Could not process record",
+                      :exception => ex,
+                      :exceptionClassName => ex.class.name,
+                      :exceptionMessage => ex.message,
+                      :exceptionBacktrace => ex.backtrace,
+                      :exceptionCause => ex.cause,
+                      :exceptionInspect => ex.inspect,
+                      :record => e.to_s)
+
         @parse_error_count += 1
       end
     end
@@ -196,11 +205,15 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
       url_send_count_sum[url] = 0
     end
 
+    flush_error_count_sum = 0
+
     @buffer_items.each do |buffer_item|
       buffer_url_send_count = buffer_item.url_send_count
       buffer_url_send_count.each do |url, count|
         url_send_count_sum[url] += count
       end
+
+      flush_error_count_sum += buffer_item.flush_error_count
     end
 
     total_send_count = 0
@@ -216,6 +229,7 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
                  :receive_count => @receive_count,
                  :send_count => total_send_count,
                  :parse_error_count => @parse_error_count,
+                 :flush_error_count => flush_error_count_sum,
                  :url_send_count => url_send_count_sum)
     @logger.info("Filebeat status Report: #{format_filebeat_report_and_clean}") if @enable_filebeat_status_report
   end
@@ -228,6 +242,7 @@ class BufferItem
 
   attr_accessor :buffer_state
   attr_accessor :url_send_count
+  attr_accessor :flush_error_count
 
   def initialize(option = {})
     @client = option[:client]
@@ -236,6 +251,7 @@ class BufferItem
     url_list.each do |url|
       @url_send_count[url] = 0
     end
+    @flush_error_count = 0
     init_url_list(url_list, option[:index])
     @logger = option[:logger]
 
@@ -254,8 +270,16 @@ class BufferItem
         @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
         return false
       end
-    rescue => e
-      @logger.warn("Send failed", :exception => e.class.name, :backtrace => e.backtrace)
+    rescue => ex
+
+      @logger.error("Send failed",
+                    :exception => ex,
+                    :exceptionClassName => ex.class.name,
+                    :exceptionMessage => ex.message,
+                    :exceptionBacktrace => ex.backtrace,
+                    :exceptionCause => ex.cause,
+                    :exceptionInspect => ex.inspect)
+
       return false
     end
     true
@@ -267,23 +291,51 @@ class BufferItem
   # 如果当前 url 发送失败, 会尝试获取列表中下一个地址进行发送, 发送失败的 url 在 3 秒内不会再尝试发送
   # 如果所有的 url 都被标记为发送失败, sleep 5 秒后重新获取
   def flush(events, final)
-    wio = StringIO.new("w")
-    gzip_io = Zlib::GzipWriter.new(wio)
-    gzip_io.write(events.to_json)
-    gzip_io.close
-    data = Base64.strict_encode64(wio.string)
-    form_data = {"data_list" => data, "gzip" => 1}
-
-    url_item = obtain_url
-
-    until do_send(form_data, url_item[:url])
-      last_url = url_item[:url]
-      # 将发送失败的 url 标记为不可用
-      disable_url(url_item)
+    begin
+      wio = StringIO.new("w")
+      gzip_io = Zlib::GzipWriter.new(wio)
+      gzip_io.write(events.to_json)
+      gzip_io.close
+      data = Base64.strict_encode64(wio.string)
+      form_data = {"data_list" => data, "gzip" => 1}
+
       url_item = obtain_url
-      @logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
+
+      until do_send(form_data, url_item[:url])
+        last_url = url_item[:url]
+        # 将发送失败的 url 标记为不可用
+        disable_url(url_item)
+        url_item = obtain_url
+        @logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
+      end
+      @url_send_count[url_item[:url]] += events.length
+    rescue => ex
+
+      if events.length == 1
+        # 打印异常现场
+        @logger.error("Flush failed",
+                      :exception => ex,
+                      :exceptionClassName => ex.class.name,
+                      :exceptionMessage => ex.message,
+                      :exceptionBacktrace => ex.backtrace,
+                      :exceptionCause => ex.cause,
+                      :exceptionInspect => ex.inspect,
+                      :events_data => events.to_s)
+
+        @flush_error_count += 1
+
+      elsif events.length > 1
+        # 降级处理: 将events 的批量发送，降级为指数级后退重试
+        pivot = events.length / 2
+        left = events.slice(0, pivot)
+        right = events.slice(pivot, events.length)
+        flush(left, final)
+        flush(right, final)
+
+        @logger.warn("Flush retry in exponential backoff!", :left_length => left.length, :right_length => right.length)
+      end
+
     end
-    @url_send_count[url_item[:url]] += events.length
   end
 
   private

data/logstash-output-sensors_analytics.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sensors_analytics'
-  s.version       = '0.1.2'
+  s.version       = '0.1.4'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'Output plugin for Sensors Analytics'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sensors_analytics
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.4
 platform: ruby
 authors:
 - Feng Jiajie
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-07 00:00:00.000000000 Z
+date: 2021-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -20,8 +20,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.99'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -37,8 +37,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -54,8 +54,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 8.0.0
   name: logstash-mixin-http_client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -71,8 +71,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.0.23
   name: stud
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -85,8 +85,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -99,8 +99,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: sinatra
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -113,8 +113,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: webrick
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -158,8 +158,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
 summary: Output plugin for Sensors Analytics