logstash-output-thinkingdata 1.1.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36f24c7196c243ee0ec3df9f446f98e60c48d860e6404d6e38897ba2841a99fb
-  data.tar.gz: 9b9fbb9076a958cda52e69775bf23fdfd947c6925ede50cdd86f6d8138f58a2c
+  metadata.gz: e867be173142072e9d2355704097e7440cbc159c2a1a7de39f8dc97e6d2e60df
+  data.tar.gz: 92bd1a49b9f661d64370b2ba2e1fdbb6005e98dfa66850feb649113582a1f687
 SHA512:
-  metadata.gz: 2cc68f4e54b11095e2226772967499e4097175a0d4647f32b02eda64b6f5ecfb29f15d7d6e500b6cf92f903e914bdb7880611f713882e7b4a34e7347432a3f19
-  data.tar.gz: d60a81599ec0cf34c697676a85ed6d249e5de8f8d3da74adf6afcdaa42bf01b7e6038d08d8de990478872c023db548df1bafacc243d7c9133e0b9a408cbcf056
+  metadata.gz: eae1bf5a3dc7e72d02610ff020d3541df9a64a39504699f6dba1e4e61e0de429c45b7322693cd88f8f0b5016e254a8e0452fb6ab4454b5529ea8597fd6de3dd4
+  data.tar.gz: 73ed46b15a675a0d2b8ae7f1dbb897cb05147a7ac12a72552f944ffd366bc820b5c01ed75e5238056e9fff70bd9caec2deb728dab846e57b467070545d0698ce

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+**v1.2.1** (2023-07-26)
+  - 增加对message中的数据进行格式校验
+
+**v1.2.0** (2023-04-25)
+  - 支持message中传多条数据
+
 **v1.1.1** (2021-07-10)
   - 增加json格式校验
 

data/lib/logstash/outputs/thinkingdata.rb

@@ -6,7 +6,6 @@ require "stud/buffer"
 require "zlib"
 require "json"
 
-
 # An thinkingdata output that does nothing.
 class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
 
@@ -42,7 +41,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   # 是否检测appid
   config :appid_check, :validate => :boolean, :default => false
 
-  PLUGIN_VERSION = "1.1.1"
+  PLUGIN_VERSION = "1.2.1"
 
   public
 
@@ -67,9 +66,9 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
     @last_report_count = 0
     @total_send_count = 0
     buffer_config = {
-        :max_items => @flush_batch_size.to_i,
-        :max_interval => @flush_interval_sec.to_i,
-        :logger => @logger
+      :max_items => @flush_batch_size.to_i,
+      :max_interval => @flush_interval_sec.to_i,
+      :logger => @logger
     }
     buffer_initialize(buffer_config)
     @filebeat_status = {} if @is_filebeat_status_record
@@ -83,7 +82,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
 
   # def register
 
-  #验证appid
+  # 验证appid
 
   private
 
@@ -99,6 +98,25 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
     end
   end
 
+  def send_content(content, event)
+    if content.to_s[0, 1] != '{'
+      @logger.error("data error: " + content.to_s)
+      @parse_error_count += 1
+      return
+    end
+    content['#uuid'] = SecureRandom.uuid if @uuid
+    if is_filebeat_input?(event) # filebeat input 记录
+      host = event.get("[host][name]")
+      file = event.get("[log][file][path]")
+      file = event.get("[source]") if file.nil?
+      offset = event.get("[log][offset]")
+      offset = event.get("[offset]") if offset.nil?
+      log_detail = "host: #{host}, file: #{file}"
+      record_filebeat_status(log_detail, offset) if @is_filebeat_status_record
+    end
+    buffer_receive(content)
+  end
+
   public
 
   def multi_receive(events)
@@ -106,18 +124,22 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
     @receive_count += events.length
     events.each do |event|
       begin
-        content = JSON.parse(event.get("message"))
-        content['#uuid'] = SecureRandom.uuid if @uuid
-        if is_filebeat_input?(event) #filebeat input 记录
-          host = event.get("[host][name]")
-          file = event.get("[log][file][path]")
-          file = event.get("[source]") if file.nil?
-          offset = event.get("[log][offset]")
-          offset = event.get("[offset]") if offset.nil?
-          log_detail = "host: #{host}, file: #{file}"
-          record_filebeat_status(log_detail, offset) if @is_filebeat_status_record
+        message = event.get("message")
+        # 判断 message 中的数据是否为json array
+        if message[0, 1] == "["
+          contents = JSON.parse(message)
+          contents.each do |content|
+            begin
+              send_content(content, event)
+            rescue => e
+              @logger.error("Could not process content", :content => event.to_s, :Exception => e)
+              @parse_error_count += 1
+            end
+          end
+        else
+          content = JSON.parse(message)
+          send_content(content, event)
         end
-        buffer_receive(content)
       rescue => e
         @logger.error("Could not process content", :content => event.to_s, :Exception => e)
         @parse_error_count += 1
@@ -154,16 +176,16 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
       compress_type = 'gzip'
     end
     if @appid.nil? || @appid.empty?
-      headers = {'custom_appid' => 'true', 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
-                 'compress' => compress_type, 'TA-Integration-Type' => 'logstash',
-                 'TA-Integration-Version' => PLUGIN_VERSION, 'TA-Integration-Count' => events.length.to_s}
+      headers = { 'custom_appid' => 'true', 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
+                  'compress' => compress_type, 'TA-Integration-Type' => 'logstash',
+                  'TA-Integration-Version' => PLUGIN_VERSION, 'TA-Integration-Count' => events.length.to_s }
     else
-      headers = {'appid' => @appid, 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
-                 'compress' => compress_type, 'TA-Integration-Type' => 'logstash',
-                 'TA-Integration-Version' => PLUGIN_VERSION, 'TA-Integration-Count' => events.length.to_s}
+      headers = { 'appid' => @appid, 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
+                  'compress' => compress_type, 'TA-Integration-Type' => 'logstash',
+                  'TA-Integration-Version' => PLUGIN_VERSION, 'TA-Integration-Count' => events.length.to_s }
     end
 
-    until do_send(data, headers)
+    until do_send(data, events, headers)
       sleep 5
     end
     @total_send_count += events.length
@@ -171,12 +193,17 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
 
   private
 
-  def do_send(data, headers)
+  def do_send(data, events, headers)
     begin
       response = @client.post(@url, :body => data, :headers => headers).call
       if response.code != 200
         @logger.error("Send failed, code: #{response.code}, body: #{response.body}", :url => @url)
         return false
+      else
+        response_body = JSON.parse(response.body)
+        if response_body["code"] != 0
+          @logger.error("Send failed, code: #{response.code}, body: #{response.body}, request data: #{events.to_s}", :url => @url)
+        end
       end
     rescue => e
       @logger.error("Send failed", :url => @url, :exception => e.class.name, :backtrace => e.backtrace)
@@ -202,7 +229,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   def record_filebeat_status(log_detail, offset)
     status = @filebeat_status[log_detail]
     if status.nil?
-      status = {:receive_time => Time.now, :offset => offset}
+      status = { :receive_time => Time.now, :offset => offset }
       @filebeat_status[log_detail] = status
     else
       status[:offset] = offset

data/logstash-output-thinkingdata.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-thinkingdata'
-  s.version       = '1.1.1'
+  s.version       = '1.2.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Output plugin for Thinkingdata Analytics'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-thinkingdata
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.1
 platform: ruby
 authors:
 - sdk
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-10 00:00:00.000000000 Z
+date: 2023-07-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -144,7 +144,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -159,8 +159,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Output plugin for Thinkingdata Analytics
 test_files: