logstash-output-thinkingdata 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6859f106a23f8b6f24120c2d51905b7b09e0c9b713235319b0c71f60c105325b
-  data.tar.gz: 50fe28c3a55a3029473afbe70ba115d0e38aa7b17da1c3acacd3572b6e4915a9
+  metadata.gz: ca56f25a56609a0ba791c30b03f9f3f2cff37d09c7ae7f708e16c43550b00f68
+  data.tar.gz: 3fda7fe64d91e3d9d889802d9fea3badc1c604bcbccb6c47c021cfd5d802ae5e
 SHA512:
-  metadata.gz: dda99ed3e2c10d2c7b86e09e71b01f1074609829791d8cb1b8fd42daf54c173cb72d3f644b007b7592d3652e8068cfd2cc6179ff25f80909c1bec292a68462e1
-  data.tar.gz: 3fe885fd3bcd52ef37442d52fbbe37a6b37835d80101f0afb179ecd1b3034e50834436f45dfe6248d5d7b4b511fda4e6b6ee4bdf95d6519641008361c35ce722
+  metadata.gz: fe27486b4018eb88f8213bd652d19820bbdaf2d1dbfeecb6341a03537a421b666c4efff7dad8bec194067174150f56f385deec77c2cb00c3f82721c6fd5fc531
+  data.tar.gz: 2e574076f997d2d0af39222558930956443acc742929b4f142547192bb9d4f51f3707debba8d6530cb6115bd736c1930d8d641af8c783b9d0fb5f3ce36fe8266

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+**v1.0.1** (2020-12-21)
+  - 修复浮点数转化为科学计数法的问题
+
 **v1.0.0** (2020-06-09)
   - 接收Logstash中event传递的message发送到TA中
   - filebeat 日志打印支持

data/lib/logstash/filter/sensors/sensors_data.rb

@@ -0,0 +1,148 @@
+require 'json'
+require 'time'
+
+class TaDataDO
+
+  attr_accessor :account_id, :distinct_id, :uuid, :type, :time, :ip, :event_name, :event_id, :first_check_id, :properties
+
+  def to_json(*a)
+
+    json_obj = {
+        "#account_id" => @account_id,
+        "#distinct_id" => @distinct_id,
+        "#uuid" => @uuid,
+        "#type" => @type,
+        "#time" => @time,
+        "#ip" => @ip,
+        "#event_name" => @event_name,
+        "#event_id" => @event_id,
+        "#first_check_id" => @first_check_id,
+        "properties" => @properties,
+    }
+    json_obj.each do |key, value|
+      if value.nil? || value == ''
+        json_obj.delete(key)
+      end
+    end
+    json_obj.to_json(*a)
+  end
+
+end
+
+def filter(event)
+  #在这里处理业务数据，如果没有进行grok等一系列处理的情况下，直接在message中获取元数据进行处理
+  begin
+    _message = event.get('message') #message 是你的上传的每条日志
+    sensors_data = JSON.parse(_message)
+
+    distinct_id = sensors_data['distinct_id']
+    original_id = sensors_data['original_id']
+    type = sensors_data['type']
+    time = sensors_data['time']
+    event_name = sensors_data['event']
+    properties = sensors_data['properties']
+
+    if properties.nil?
+      properties = {}
+    end
+
+    data = TaDataDO.new
+
+    if time.nil?
+      time = Time.now.strftime('%Y-%m-%d %H:%M:%S')
+    else
+      time = Time.at(time / 1000).strftime('%Y-%m-%d %H:%M:%S')
+    end
+    data.time = time
+
+    if (type == 'track' && event_name == '$SignUp') || type == 'track_signup'
+      data.account_id = distinct_id
+      data.distinct_id = properties['$track_signup_original_id']
+      if distinct_id == original_id
+        puts 'original_id error:' + _message + "\n"
+      end
+    elsif type == 'track' || type.index('profile_') == 0
+      is_login_id = properties['$is_login_id']
+      if is_login_id
+        data.account_id = distinct_id
+        if distinct_id != original_id
+          data.distinct_id = original_id
+        end
+      else
+        data.distinct_id = distinct_id
+      end
+    else
+      puts 'not recognized type: ' + _message + "\n"
+    end
+
+    if type == 'track' || type == 'track_signup'
+      event_name = event_name.gsub('$', '')
+      ip = properties['$ip']
+      if ip
+        data.ip = ip
+      end
+      data.type = 'track'
+      data.event_name = event_name
+    elsif type == 'profile_set'
+      data.type = 'user_set'
+    elsif type == 'profile_increment'
+      data.type = 'user_add'
+    elsif type == 'profile_delete'
+      data.type = 'user_del'
+    elsif type == 'profile_unset'
+      data.type = 'user_unset'
+    elsif type == 'profile_set_once'
+      data.type = 'user_setOnce'
+    else
+      puts '暂不支持的type: ' + type + "\n"
+    end
+
+    properties_json = {}
+    properties.each do |key, value|
+      if TA_PRESET_COLUMN_MAP.has_key?(key) && EVENT_TRACK_TYPE_SET.include?(type)
+        properties_json[TA_PRESET_COLUMN_MAP[key]] = value
+      else
+        properties_json[key.gsub('$', '')] = value
+      end
+    end
+    data.properties = properties_json
+
+    event.set('message', data.to_json)
+
+    return [event]
+  rescue Exception => e
+    return [event]
+  end
+end
+
+EVENT_TRACK_TYPE_SET = Set['track', 'track_signup']
+
+TA_PRESET_COLUMN_MAP = Hash['$ip' => '#ip',
+                            '$country' => '#country',
+                            '$province' => '#province',
+                            '$city' => '#city',
+                            '$os_version' => '#os_version',
+                            '$manufacturer' => '#manufacturer',
+                            '$lib_version' => '#lib_version',
+                            '$os' => '#os',
+                            '$device_id' => '#device_id',
+                            '$screen_height' => '#screen_height',
+                            '$model' => '#device_model',
+                            '$app_version' => '#app_version',
+                            '$screen_width' => '#screen_width',
+                            '$lib' => '#lib',
+                            '$network_type' => '#network_type',
+                            '$carrier' => '#carrier',
+                            '$browser' => '#browser',
+                            '$browser_version' => '#browser_version',
+                            'event_duration' => '#duration',
+                            '$url_path' => '#url_path',
+                            '$title' => '#title',
+                            '$url' => '#url',
+                            '$element_content' => '#element_content',
+                            '$element_id' => '#element_id',
+                            '$element_type' => '#element_type',
+                            '$element_selector' => '#element_selector',
+                            '$scene' => '#scene',
+                            '$referrer' => '#referrer',
+                            '$share_depth' => '#share_depth']

data/lib/logstash/outputs/thinkingdata.rb

@@ -22,7 +22,10 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   config :url, :validate => :string, :required => :true
 
   # 数据的项目appid
-  config :appid, :validate => :string, :required => :true
+  config :appid, :validate => :string
+
+  # 是否将appid封装于数据中
+  config :is_sync_data, :validate => :boolean, :default => false
 
   # 触发 flush 间隔时间
   config :flush_interval_sec, :validate => :number, :default => 2
@@ -33,26 +36,27 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   # 是否压缩数据 0:不压缩，1:gzip压缩
   config :compress, :validate => :number, :default => 1
 
-  #是否开启uuid
+  # 是否开启uuid
   config :uuid, :validate => :boolean, :default => false
 
-
   # 开启 filebeat 状态记录
   config :is_filebeat_status_record, :validate => :boolean, :default => true
 
   # 是否检测appid
   config :appid_check, :validate => :boolean, :default => false
 
-  PLUGIN_VERSION = "1.0.0"
+  PLUGIN_VERSION = "1.0.1"
 
   public
+
   def register
     @logger.info("Registering thinkingdata Output",
                  :url => @url,
                  :appid => @appid,
+                 :is_sync_data => @is_sync_data,
                  :flush_interval_sec => @flush_interval_sec,
                  :flush_batch_size => @flush_batch_size,
-                 :compress =>@compress,
+                 :compress => @compress,
                  :uuid => @uuid,
                  :is_filebeat_status_record => @is_filebeat_status_record,
                  :appid_check => @appid_check
@@ -79,14 +83,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
         report
       end
     end
-  end # def register
+  end
+
+  # def register
 
   #验证appid
+
   private
+
   def ta_appid_check
     @server_uri = URI(@url)
     @server_uri.path = "/check_appid"
-    @server_uri.query= "appid="+@appid
+    @server_uri.query = "appid=" + @appid
     response = client.get(@server_uri.to_s).call
     result = JSON.parse(response.body)
     if result['code'] == -2
@@ -96,17 +104,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   end
 
   public
+
   def multi_receive(events)
     return if events.empty?
     @receive_count += events.length
     events.each do |event|
       begin
-        content = LogStash::Json.load(event.get("message"))
+        content = JSON.parse(event.get("message"))
         content['#uuid'] = SecureRandom.uuid if @uuid
-        if is_filebeat_input?(event)   #filebeat input 记录
+        if is_filebeat_input?(event) #filebeat input 记录
           host = event.get("[host][name]")
           file = event.get("[log][file][path]")
-          file = event.get("[source]")if file.nil?
+          file = event.get("[source]") if file.nil?
           offset = event.get("[log][offset]")
           offset = event.get("[offset]") if offset.nil?
           log_detail = "host: #{host}, file: #{file}"
@@ -118,9 +127,12 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
         @parse_error_count += 1
       end
     end
-  end # def event
+  end
+
+  # def event
 
   public
+
   def close
     buffer_state[:timer].kill
     buffer_flush(:final => true)
@@ -130,43 +142,46 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   end
 
   public
+
   def flush(events, final)
     if @compress == 0
-       data = events.to_json
-       compress_type = 'none'
+      data = events.to_json
+      compress_type = 'none'
     else
-       gz = StringIO.new("w")
-       gz.set_encoding("BINARY")
-       z = Zlib::GzipWriter.new(gz)
-       z.write(events.to_json)
-       z.close
-       data = gz.string
-       compress_type = 'gzip'
+      gz = StringIO.new("w")
+      gz.set_encoding("BINARY")
+      z = Zlib::GzipWriter.new(gz)
+      z.write(events.to_json)
+      z.close
+      data = gz.string
+      compress_type = 'gzip'
+    end
+    headers = {'appid' => @appid, 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
+               'compress' => compress_type}
+    until do_send(data, headers)
+      sleep 5
     end
-      headers ={'appid' => @appid,'version'=>PLUGIN_VERSION,'user-agent' =>'logstash_'+ PLUGIN_VERSION,
-              'compress' => compress_type}
-     until do_send(data,headers)
-       sleep 5
-     end
     @total_send_count += events.length
   end
 
   private
+
   def do_send(data, headers)
     begin
-      response = @client.post(@url, :body => data,:headers =>headers).call
+      response = @client.post(@url, :body => data, :headers => headers).call
       if response.code != 200
-        @logger.error("Send failed, code: #{response.code}, body: #{response.body}",:url => @url)
-         return false
+        @logger.error("Send failed, code: #{response.code}, body: #{response.body}", :url => @url)
+        return false
       end
     rescue => e
-      @logger.error("Send failed",:url => @url, :exception => e.class.name, :backtrace => e.backtrace)
+      @logger.error("Send failed", :url => @url, :exception => e.class.name, :backtrace => e.backtrace)
       return false
     end
     true
   end
 
   private
+
   def is_filebeat_input?(event)
     type = event.get("[agent][type]")
     return true if !type.nil? && type == "filebeat"
@@ -178,6 +193,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   end
 
   private
+
   def record_filebeat_status(log_detail, offset)
     status = @filebeat_status[log_detail]
     if status.nil?
@@ -190,15 +206,16 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
   end
 
   private
+
   def report
-     interval_count = @total_send_count - @last_report_count
-     @last_report_count = @total_send_count
-     @logger.info("Report",
-                  "IntervalReceive(records):" => interval_count,
-                  "Receive(records):" => @receive_count,
-                  "TotalSend(records)" => @total_send_count,
-                  "ParseError(records)" => @parse_error_count)
-     @logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status !={}
+    interval_count = @total_send_count - @last_report_count
+    @last_report_count = @total_send_count
+    @logger.info("Report:",
+                 "IntervalReceive(records):" => interval_count,
+                 "Receive(records):" => @receive_count,
+                 "TotalSend(records):" => @total_send_count,
+                 "ParseError(records):" => @parse_error_count)
+    @logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status != {}
   end
 
   private
@@ -212,4 +229,6 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
     result
   end
 
-end # class LogStash::Outputs::Thinkingdata
+end
+
+# class LogStash::Outputs::Thinkingdata

data/logstash-output-thinkingdata.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-thinkingdata'
-  s.version       = '1.0.0'
+  s.version       = '1.0.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Output plugin for Thinkingdata Analytics'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-thinkingdata
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - sdk
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-09 00:00:00.000000000 Z
+date: 2020-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -134,6 +134,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/logstash/filter/sensors/sensors_data.rb
 - lib/logstash/outputs/thinkingdata.rb
 - logstash-output-thinkingdata.gemspec
 - spec/outputs/thinkingdata_spec.rb
@@ -143,7 +144,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -158,8 +159,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.0.8
+signing_key:
 specification_version: 4
 summary: Output plugin for Thinkingdata Analytics
 test_files: