logstash-output-thinkingdata 1.0.0 → 1.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6859f106a23f8b6f24120c2d51905b7b09e0c9b713235319b0c71f60c105325b
4
- data.tar.gz: 50fe28c3a55a3029473afbe70ba115d0e38aa7b17da1c3acacd3572b6e4915a9
3
+ metadata.gz: ca56f25a56609a0ba791c30b03f9f3f2cff37d09c7ae7f708e16c43550b00f68
4
+ data.tar.gz: 3fda7fe64d91e3d9d889802d9fea3badc1c604bcbccb6c47c021cfd5d802ae5e
5
5
  SHA512:
6
- metadata.gz: dda99ed3e2c10d2c7b86e09e71b01f1074609829791d8cb1b8fd42daf54c173cb72d3f644b007b7592d3652e8068cfd2cc6179ff25f80909c1bec292a68462e1
7
- data.tar.gz: 3fe885fd3bcd52ef37442d52fbbe37a6b37835d80101f0afb179ecd1b3034e50834436f45dfe6248d5d7b4b511fda4e6b6ee4bdf95d6519641008361c35ce722
6
+ metadata.gz: fe27486b4018eb88f8213bd652d19820bbdaf2d1dbfeecb6341a03537a421b666c4efff7dad8bec194067174150f56f385deec77c2cb00c3f82721c6fd5fc531
7
+ data.tar.gz: 2e574076f997d2d0af39222558930956443acc742929b4f142547192bb9d4f51f3707debba8d6530cb6115bd736c1930d8d641af8c783b9d0fb5f3ce36fe8266
@@ -1,3 +1,6 @@
1
+ **v1.0.1** (2020-12-21)
2
+ - 修复浮点数转化为科学计数法的问题
3
+
1
4
  **v1.0.0** (2020-06-09)
2
5
  - 接收Logstash中event传递的message发送到TA中
3
6
  - filebeat 日志打印支持
@@ -0,0 +1,148 @@
1
+ require 'json'
2
+ require 'time'
3
+
4
+ class TaDataDO
5
+
6
+ attr_accessor :account_id, :distinct_id, :uuid, :type, :time, :ip, :event_name, :event_id, :first_check_id, :properties
7
+
8
+ def to_json(*a)
9
+
10
+ json_obj = {
11
+ "#account_id" => @account_id,
12
+ "#distinct_id" => @distinct_id,
13
+ "#uuid" => @uuid,
14
+ "#type" => @type,
15
+ "#time" => @time,
16
+ "#ip" => @ip,
17
+ "#event_name" => @event_name,
18
+ "#event_id" => @event_id,
19
+ "#first_check_id" => @first_check_id,
20
+ "properties" => @properties,
21
+ }
22
+ json_obj.each do |key, value|
23
+ if value.nil? || value == ''
24
+ json_obj.delete(key)
25
+ end
26
+ end
27
+ json_obj.to_json(*a)
28
+ end
29
+
30
+ end
31
+
32
+ def filter(event)
33
+ #在这里处理业务数据,如果没有进行grok等一系列处理的情况下,直接在message中获取元数据进行处理
34
+ begin
35
+ _message = event.get('message') #message 是你的上传的每条日志
36
+ sensors_data = JSON.parse(_message)
37
+
38
+ distinct_id = sensors_data['distinct_id']
39
+ original_id = sensors_data['original_id']
40
+ type = sensors_data['type']
41
+ time = sensors_data['time']
42
+ event_name = sensors_data['event']
43
+ properties = sensors_data['properties']
44
+
45
+ if properties.nil?
46
+ properties = {}
47
+ end
48
+
49
+ data = TaDataDO.new
50
+
51
+ if time.nil?
52
+ time = Time.now.strftime('%Y-%m-%d %H:%M:%S')
53
+ else
54
+ time = Time.at(time / 1000).strftime('%Y-%m-%d %H:%M:%S')
55
+ end
56
+ data.time = time
57
+
58
+ if (type == 'track' && event_name == '$SignUp') || type == 'track_signup'
59
+ data.account_id = distinct_id
60
+ data.distinct_id = properties['$track_signup_original_id']
61
+ if distinct_id == original_id
62
+ puts 'original_id error:' + _message + "\n"
63
+ end
64
+ elsif type == 'track' || type.index('profile_') == 0
65
+ is_login_id = properties['$is_login_id']
66
+ if is_login_id
67
+ data.account_id = distinct_id
68
+ if distinct_id != original_id
69
+ data.distinct_id = original_id
70
+ end
71
+ else
72
+ data.distinct_id = distinct_id
73
+ end
74
+ else
75
+ puts 'not recognized type: ' + _message + "\n"
76
+ end
77
+
78
+ if type == 'track' || type == 'track_signup'
79
+ event_name = event_name.gsub('$', '')
80
+ ip = properties['$ip']
81
+ if ip
82
+ data.ip = ip
83
+ end
84
+ data.type = 'track'
85
+ data.event_name = event_name
86
+ elsif type == 'profile_set'
87
+ data.type = 'user_set'
88
+ elsif type == 'profile_increment'
89
+ data.type = 'user_add'
90
+ elsif type == 'profile_delete'
91
+ data.type = 'user_del'
92
+ elsif type == 'profile_unset'
93
+ data.type = 'user_unset'
94
+ elsif type == 'profile_set_once'
95
+ data.type = 'user_setOnce'
96
+ else
97
+ puts '暂不支持的type: ' + type + "\n"
98
+ end
99
+
100
+ properties_json = {}
101
+ properties.each do |key, value|
102
+ if TA_PRESET_COLUMN_MAP.has_key?(key) && EVENT_TRACK_TYPE_SET.include?(type)
103
+ properties_json[TA_PRESET_COLUMN_MAP[key]] = value
104
+ else
105
+ properties_json[key.gsub('$', '')] = value
106
+ end
107
+ end
108
+ data.properties = properties_json
109
+
110
+ event.set('message', data.to_json)
111
+
112
+ return [event]
113
+ rescue Exception => e
114
+ return [event]
115
+ end
116
+ end
117
+
118
+ EVENT_TRACK_TYPE_SET = Set['track', 'track_signup']
119
+
120
+ TA_PRESET_COLUMN_MAP = Hash['$ip' => '#ip',
121
+ '$country' => '#country',
122
+ '$province' => '#province',
123
+ '$city' => '#city',
124
+ '$os_version' => '#os_version',
125
+ '$manufacturer' => '#manufacturer',
126
+ '$lib_version' => '#lib_version',
127
+ '$os' => '#os',
128
+ '$device_id' => '#device_id',
129
+ '$screen_height' => '#screen_height',
130
+ '$model' => '#device_model',
131
+ '$app_version' => '#app_version',
132
+ '$screen_width' => '#screen_width',
133
+ '$lib' => '#lib',
134
+ '$network_type' => '#network_type',
135
+ '$carrier' => '#carrier',
136
+ '$browser' => '#browser',
137
+ '$browser_version' => '#browser_version',
138
+ 'event_duration' => '#duration',
139
+ '$url_path' => '#url_path',
140
+ '$title' => '#title',
141
+ '$url' => '#url',
142
+ '$element_content' => '#element_content',
143
+ '$element_id' => '#element_id',
144
+ '$element_type' => '#element_type',
145
+ '$element_selector' => '#element_selector',
146
+ '$scene' => '#scene',
147
+ '$referrer' => '#referrer',
148
+ '$share_depth' => '#share_depth']
@@ -22,7 +22,10 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
22
22
  config :url, :validate => :string, :required => :true
23
23
 
24
24
  # 数据的项目appid
25
- config :appid, :validate => :string, :required => :true
25
+ config :appid, :validate => :string
26
+
27
+ # 是否将appid封装于数据中
28
+ config :is_sync_data, :validate => :boolean, :default => false
26
29
 
27
30
  # 触发 flush 间隔时间
28
31
  config :flush_interval_sec, :validate => :number, :default => 2
@@ -33,26 +36,27 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
33
36
  # 是否压缩数据 0:不压缩,1:gzip压缩
34
37
  config :compress, :validate => :number, :default => 1
35
38
 
36
- #是否开启uuid
39
+ # 是否开启uuid
37
40
  config :uuid, :validate => :boolean, :default => false
38
41
 
39
-
40
42
  # 开启 filebeat 状态记录
41
43
  config :is_filebeat_status_record, :validate => :boolean, :default => true
42
44
 
43
45
  # 是否检测appid
44
46
  config :appid_check, :validate => :boolean, :default => false
45
47
 
46
- PLUGIN_VERSION = "1.0.0"
48
+ PLUGIN_VERSION = "1.0.1"
47
49
 
48
50
  public
51
+
49
52
  def register
50
53
  @logger.info("Registering thinkingdata Output",
51
54
  :url => @url,
52
55
  :appid => @appid,
56
+ :is_sync_data => @is_sync_data,
53
57
  :flush_interval_sec => @flush_interval_sec,
54
58
  :flush_batch_size => @flush_batch_size,
55
- :compress =>@compress,
59
+ :compress => @compress,
56
60
  :uuid => @uuid,
57
61
  :is_filebeat_status_record => @is_filebeat_status_record,
58
62
  :appid_check => @appid_check
@@ -79,14 +83,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
79
83
  report
80
84
  end
81
85
  end
82
- end # def register
86
+ end
87
+
88
+ # def register
83
89
 
84
90
  #验证appid
91
+
85
92
  private
93
+
86
94
  def ta_appid_check
87
95
  @server_uri = URI(@url)
88
96
  @server_uri.path = "/check_appid"
89
- @server_uri.query= "appid="+@appid
97
+ @server_uri.query = "appid=" + @appid
90
98
  response = client.get(@server_uri.to_s).call
91
99
  result = JSON.parse(response.body)
92
100
  if result['code'] == -2
@@ -96,17 +104,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
96
104
  end
97
105
 
98
106
  public
107
+
99
108
  def multi_receive(events)
100
109
  return if events.empty?
101
110
  @receive_count += events.length
102
111
  events.each do |event|
103
112
  begin
104
- content = LogStash::Json.load(event.get("message"))
113
+ content = JSON.parse(event.get("message"))
105
114
  content['#uuid'] = SecureRandom.uuid if @uuid
106
- if is_filebeat_input?(event) #filebeat input 记录
115
+ if is_filebeat_input?(event) #filebeat input 记录
107
116
  host = event.get("[host][name]")
108
117
  file = event.get("[log][file][path]")
109
- file = event.get("[source]")if file.nil?
118
+ file = event.get("[source]") if file.nil?
110
119
  offset = event.get("[log][offset]")
111
120
  offset = event.get("[offset]") if offset.nil?
112
121
  log_detail = "host: #{host}, file: #{file}"
@@ -118,9 +127,12 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
118
127
  @parse_error_count += 1
119
128
  end
120
129
  end
121
- end # def event
130
+ end
131
+
132
+ # def event
122
133
 
123
134
  public
135
+
124
136
  def close
125
137
  buffer_state[:timer].kill
126
138
  buffer_flush(:final => true)
@@ -130,43 +142,46 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
130
142
  end
131
143
 
132
144
  public
145
+
133
146
  def flush(events, final)
134
147
  if @compress == 0
135
- data = events.to_json
136
- compress_type = 'none'
148
+ data = events.to_json
149
+ compress_type = 'none'
137
150
  else
138
- gz = StringIO.new("w")
139
- gz.set_encoding("BINARY")
140
- z = Zlib::GzipWriter.new(gz)
141
- z.write(events.to_json)
142
- z.close
143
- data = gz.string
144
- compress_type = 'gzip'
151
+ gz = StringIO.new("w")
152
+ gz.set_encoding("BINARY")
153
+ z = Zlib::GzipWriter.new(gz)
154
+ z.write(events.to_json)
155
+ z.close
156
+ data = gz.string
157
+ compress_type = 'gzip'
158
+ end
159
+ headers = {'appid' => @appid, 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
160
+ 'compress' => compress_type}
161
+ until do_send(data, headers)
162
+ sleep 5
145
163
  end
146
- headers ={'appid' => @appid,'version'=>PLUGIN_VERSION,'user-agent' =>'logstash_'+ PLUGIN_VERSION,
147
- 'compress' => compress_type}
148
- until do_send(data,headers)
149
- sleep 5
150
- end
151
164
  @total_send_count += events.length
152
165
  end
153
166
 
154
167
  private
168
+
155
169
  def do_send(data, headers)
156
170
  begin
157
- response = @client.post(@url, :body => data,:headers =>headers).call
171
+ response = @client.post(@url, :body => data, :headers => headers).call
158
172
  if response.code != 200
159
- @logger.error("Send failed, code: #{response.code}, body: #{response.body}",:url => @url)
160
- return false
173
+ @logger.error("Send failed, code: #{response.code}, body: #{response.body}", :url => @url)
174
+ return false
161
175
  end
162
176
  rescue => e
163
- @logger.error("Send failed",:url => @url, :exception => e.class.name, :backtrace => e.backtrace)
177
+ @logger.error("Send failed", :url => @url, :exception => e.class.name, :backtrace => e.backtrace)
164
178
  return false
165
179
  end
166
180
  true
167
181
  end
168
182
 
169
183
  private
184
+
170
185
  def is_filebeat_input?(event)
171
186
  type = event.get("[agent][type]")
172
187
  return true if !type.nil? && type == "filebeat"
@@ -178,6 +193,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
178
193
  end
179
194
 
180
195
  private
196
+
181
197
  def record_filebeat_status(log_detail, offset)
182
198
  status = @filebeat_status[log_detail]
183
199
  if status.nil?
@@ -190,15 +206,16 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
190
206
  end
191
207
 
192
208
  private
209
+
193
210
  def report
194
- interval_count = @total_send_count - @last_report_count
195
- @last_report_count = @total_send_count
196
- @logger.info("Report",
197
- "IntervalReceive(records):" => interval_count,
198
- "Receive(records):" => @receive_count,
199
- "TotalSend(records)" => @total_send_count,
200
- "ParseError(records)" => @parse_error_count)
201
- @logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status !={}
211
+ interval_count = @total_send_count - @last_report_count
212
+ @last_report_count = @total_send_count
213
+ @logger.info("Report:",
214
+ "IntervalReceive(records):" => interval_count,
215
+ "Receive(records):" => @receive_count,
216
+ "TotalSend(records):" => @total_send_count,
217
+ "ParseError(records):" => @parse_error_count)
218
+ @logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status != {}
202
219
  end
203
220
 
204
221
  private
@@ -212,4 +229,6 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
212
229
  result
213
230
  end
214
231
 
215
- end # class LogStash::Outputs::Thinkingdata
232
+ end
233
+
234
+ # class LogStash::Outputs::Thinkingdata
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-thinkingdata'
3
- s.version = '1.0.0'
3
+ s.version = '1.0.1'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = 'Output plugin for Thinkingdata Analytics'
6
6
  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-thinkingdata
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.0
4
+ version: 1.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - sdk
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-06-09 00:00:00.000000000 Z
11
+ date: 2020-12-21 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core-plugin-api
@@ -134,6 +134,7 @@ files:
134
134
  - Gemfile
135
135
  - LICENSE
136
136
  - README.md
137
+ - lib/logstash/filter/sensors/sensors_data.rb
137
138
  - lib/logstash/outputs/thinkingdata.rb
138
139
  - logstash-output-thinkingdata.gemspec
139
140
  - spec/outputs/thinkingdata_spec.rb
@@ -143,7 +144,7 @@ licenses:
143
144
  metadata:
144
145
  logstash_plugin: 'true'
145
146
  logstash_group: output
146
- post_install_message:
147
+ post_install_message:
147
148
  rdoc_options: []
148
149
  require_paths:
149
150
  - lib
@@ -158,8 +159,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
158
159
  - !ruby/object:Gem::Version
159
160
  version: '0'
160
161
  requirements: []
161
- rubygems_version: 3.0.3
162
- signing_key:
162
+ rubygems_version: 3.0.8
163
+ signing_key:
163
164
  specification_version: 4
164
165
  summary: Output plugin for Thinkingdata Analytics
165
166
  test_files: