logstash-output-thinkingdata 1.0.0 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/filter/sensors/sensors_data.rb +148 -0
- data/lib/logstash/outputs/thinkingdata.rb +57 -38
- data/logstash-output-thinkingdata.gemspec +1 -1
- metadata +7 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ca56f25a56609a0ba791c30b03f9f3f2cff37d09c7ae7f708e16c43550b00f68
|
4
|
+
data.tar.gz: 3fda7fe64d91e3d9d889802d9fea3badc1c604bcbccb6c47c021cfd5d802ae5e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fe27486b4018eb88f8213bd652d19820bbdaf2d1dbfeecb6341a03537a421b666c4efff7dad8bec194067174150f56f385deec77c2cb00c3f82721c6fd5fc531
|
7
|
+
data.tar.gz: 2e574076f997d2d0af39222558930956443acc742929b4f142547192bb9d4f51f3707debba8d6530cb6115bd736c1930d8d641af8c783b9d0fb5f3ce36fe8266
|
data/CHANGELOG.md
CHANGED
@@ -0,0 +1,148 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
class TaDataDO
|
5
|
+
|
6
|
+
attr_accessor :account_id, :distinct_id, :uuid, :type, :time, :ip, :event_name, :event_id, :first_check_id, :properties
|
7
|
+
|
8
|
+
def to_json(*a)
|
9
|
+
|
10
|
+
json_obj = {
|
11
|
+
"#account_id" => @account_id,
|
12
|
+
"#distinct_id" => @distinct_id,
|
13
|
+
"#uuid" => @uuid,
|
14
|
+
"#type" => @type,
|
15
|
+
"#time" => @time,
|
16
|
+
"#ip" => @ip,
|
17
|
+
"#event_name" => @event_name,
|
18
|
+
"#event_id" => @event_id,
|
19
|
+
"#first_check_id" => @first_check_id,
|
20
|
+
"properties" => @properties,
|
21
|
+
}
|
22
|
+
json_obj.each do |key, value|
|
23
|
+
if value.nil? || value == ''
|
24
|
+
json_obj.delete(key)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
json_obj.to_json(*a)
|
28
|
+
end
|
29
|
+
|
30
|
+
end
|
31
|
+
|
32
|
+
def filter(event)
|
33
|
+
#在这里处理业务数据,如果没有进行grok等一系列处理的情况下,直接在message中获取元数据进行处理
|
34
|
+
begin
|
35
|
+
_message = event.get('message') #message 是你的上传的每条日志
|
36
|
+
sensors_data = JSON.parse(_message)
|
37
|
+
|
38
|
+
distinct_id = sensors_data['distinct_id']
|
39
|
+
original_id = sensors_data['original_id']
|
40
|
+
type = sensors_data['type']
|
41
|
+
time = sensors_data['time']
|
42
|
+
event_name = sensors_data['event']
|
43
|
+
properties = sensors_data['properties']
|
44
|
+
|
45
|
+
if properties.nil?
|
46
|
+
properties = {}
|
47
|
+
end
|
48
|
+
|
49
|
+
data = TaDataDO.new
|
50
|
+
|
51
|
+
if time.nil?
|
52
|
+
time = Time.now.strftime('%Y-%m-%d %H:%M:%S')
|
53
|
+
else
|
54
|
+
time = Time.at(time / 1000).strftime('%Y-%m-%d %H:%M:%S')
|
55
|
+
end
|
56
|
+
data.time = time
|
57
|
+
|
58
|
+
if (type == 'track' && event_name == '$SignUp') || type == 'track_signup'
|
59
|
+
data.account_id = distinct_id
|
60
|
+
data.distinct_id = properties['$track_signup_original_id']
|
61
|
+
if distinct_id == original_id
|
62
|
+
puts 'original_id error:' + _message + "\n"
|
63
|
+
end
|
64
|
+
elsif type == 'track' || type.index('profile_') == 0
|
65
|
+
is_login_id = properties['$is_login_id']
|
66
|
+
if is_login_id
|
67
|
+
data.account_id = distinct_id
|
68
|
+
if distinct_id != original_id
|
69
|
+
data.distinct_id = original_id
|
70
|
+
end
|
71
|
+
else
|
72
|
+
data.distinct_id = distinct_id
|
73
|
+
end
|
74
|
+
else
|
75
|
+
puts 'not recognized type: ' + _message + "\n"
|
76
|
+
end
|
77
|
+
|
78
|
+
if type == 'track' || type == 'track_signup'
|
79
|
+
event_name = event_name.gsub('$', '')
|
80
|
+
ip = properties['$ip']
|
81
|
+
if ip
|
82
|
+
data.ip = ip
|
83
|
+
end
|
84
|
+
data.type = 'track'
|
85
|
+
data.event_name = event_name
|
86
|
+
elsif type == 'profile_set'
|
87
|
+
data.type = 'user_set'
|
88
|
+
elsif type == 'profile_increment'
|
89
|
+
data.type = 'user_add'
|
90
|
+
elsif type == 'profile_delete'
|
91
|
+
data.type = 'user_del'
|
92
|
+
elsif type == 'profile_unset'
|
93
|
+
data.type = 'user_unset'
|
94
|
+
elsif type == 'profile_set_once'
|
95
|
+
data.type = 'user_setOnce'
|
96
|
+
else
|
97
|
+
puts '暂不支持的type: ' + type + "\n"
|
98
|
+
end
|
99
|
+
|
100
|
+
properties_json = {}
|
101
|
+
properties.each do |key, value|
|
102
|
+
if TA_PRESET_COLUMN_MAP.has_key?(key) && EVENT_TRACK_TYPE_SET.include?(type)
|
103
|
+
properties_json[TA_PRESET_COLUMN_MAP[key]] = value
|
104
|
+
else
|
105
|
+
properties_json[key.gsub('$', '')] = value
|
106
|
+
end
|
107
|
+
end
|
108
|
+
data.properties = properties_json
|
109
|
+
|
110
|
+
event.set('message', data.to_json)
|
111
|
+
|
112
|
+
return [event]
|
113
|
+
rescue Exception => e
|
114
|
+
return [event]
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
EVENT_TRACK_TYPE_SET = Set['track', 'track_signup']
|
119
|
+
|
120
|
+
TA_PRESET_COLUMN_MAP = Hash['$ip' => '#ip',
|
121
|
+
'$country' => '#country',
|
122
|
+
'$province' => '#province',
|
123
|
+
'$city' => '#city',
|
124
|
+
'$os_version' => '#os_version',
|
125
|
+
'$manufacturer' => '#manufacturer',
|
126
|
+
'$lib_version' => '#lib_version',
|
127
|
+
'$os' => '#os',
|
128
|
+
'$device_id' => '#device_id',
|
129
|
+
'$screen_height' => '#screen_height',
|
130
|
+
'$model' => '#device_model',
|
131
|
+
'$app_version' => '#app_version',
|
132
|
+
'$screen_width' => '#screen_width',
|
133
|
+
'$lib' => '#lib',
|
134
|
+
'$network_type' => '#network_type',
|
135
|
+
'$carrier' => '#carrier',
|
136
|
+
'$browser' => '#browser',
|
137
|
+
'$browser_version' => '#browser_version',
|
138
|
+
'event_duration' => '#duration',
|
139
|
+
'$url_path' => '#url_path',
|
140
|
+
'$title' => '#title',
|
141
|
+
'$url' => '#url',
|
142
|
+
'$element_content' => '#element_content',
|
143
|
+
'$element_id' => '#element_id',
|
144
|
+
'$element_type' => '#element_type',
|
145
|
+
'$element_selector' => '#element_selector',
|
146
|
+
'$scene' => '#scene',
|
147
|
+
'$referrer' => '#referrer',
|
148
|
+
'$share_depth' => '#share_depth']
|
@@ -22,7 +22,10 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
22
22
|
config :url, :validate => :string, :required => :true
|
23
23
|
|
24
24
|
# 数据的项目appid
|
25
|
-
config :appid, :validate => :string
|
25
|
+
config :appid, :validate => :string
|
26
|
+
|
27
|
+
# 是否将appid封装于数据中
|
28
|
+
config :is_sync_data, :validate => :boolean, :default => false
|
26
29
|
|
27
30
|
# 触发 flush 间隔时间
|
28
31
|
config :flush_interval_sec, :validate => :number, :default => 2
|
@@ -33,26 +36,27 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
33
36
|
# 是否压缩数据 0:不压缩,1:gzip压缩
|
34
37
|
config :compress, :validate => :number, :default => 1
|
35
38
|
|
36
|
-
|
39
|
+
# 是否开启uuid
|
37
40
|
config :uuid, :validate => :boolean, :default => false
|
38
41
|
|
39
|
-
|
40
42
|
# 开启 filebeat 状态记录
|
41
43
|
config :is_filebeat_status_record, :validate => :boolean, :default => true
|
42
44
|
|
43
45
|
# 是否检测appid
|
44
46
|
config :appid_check, :validate => :boolean, :default => false
|
45
47
|
|
46
|
-
PLUGIN_VERSION = "1.0.
|
48
|
+
PLUGIN_VERSION = "1.0.1"
|
47
49
|
|
48
50
|
public
|
51
|
+
|
49
52
|
def register
|
50
53
|
@logger.info("Registering thinkingdata Output",
|
51
54
|
:url => @url,
|
52
55
|
:appid => @appid,
|
56
|
+
:is_sync_data => @is_sync_data,
|
53
57
|
:flush_interval_sec => @flush_interval_sec,
|
54
58
|
:flush_batch_size => @flush_batch_size,
|
55
|
-
:compress
|
59
|
+
:compress => @compress,
|
56
60
|
:uuid => @uuid,
|
57
61
|
:is_filebeat_status_record => @is_filebeat_status_record,
|
58
62
|
:appid_check => @appid_check
|
@@ -79,14 +83,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
79
83
|
report
|
80
84
|
end
|
81
85
|
end
|
82
|
-
end
|
86
|
+
end
|
87
|
+
|
88
|
+
# def register
|
83
89
|
|
84
90
|
#验证appid
|
91
|
+
|
85
92
|
private
|
93
|
+
|
86
94
|
def ta_appid_check
|
87
95
|
@server_uri = URI(@url)
|
88
96
|
@server_uri.path = "/check_appid"
|
89
|
-
@server_uri.query= "appid="
|
97
|
+
@server_uri.query = "appid=" + @appid
|
90
98
|
response = client.get(@server_uri.to_s).call
|
91
99
|
result = JSON.parse(response.body)
|
92
100
|
if result['code'] == -2
|
@@ -96,17 +104,18 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
96
104
|
end
|
97
105
|
|
98
106
|
public
|
107
|
+
|
99
108
|
def multi_receive(events)
|
100
109
|
return if events.empty?
|
101
110
|
@receive_count += events.length
|
102
111
|
events.each do |event|
|
103
112
|
begin
|
104
|
-
content =
|
113
|
+
content = JSON.parse(event.get("message"))
|
105
114
|
content['#uuid'] = SecureRandom.uuid if @uuid
|
106
|
-
if is_filebeat_input?(event)
|
115
|
+
if is_filebeat_input?(event) #filebeat input 记录
|
107
116
|
host = event.get("[host][name]")
|
108
117
|
file = event.get("[log][file][path]")
|
109
|
-
file = event.get("[source]")if file.nil?
|
118
|
+
file = event.get("[source]") if file.nil?
|
110
119
|
offset = event.get("[log][offset]")
|
111
120
|
offset = event.get("[offset]") if offset.nil?
|
112
121
|
log_detail = "host: #{host}, file: #{file}"
|
@@ -118,9 +127,12 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
118
127
|
@parse_error_count += 1
|
119
128
|
end
|
120
129
|
end
|
121
|
-
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# def event
|
122
133
|
|
123
134
|
public
|
135
|
+
|
124
136
|
def close
|
125
137
|
buffer_state[:timer].kill
|
126
138
|
buffer_flush(:final => true)
|
@@ -130,43 +142,46 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
130
142
|
end
|
131
143
|
|
132
144
|
public
|
145
|
+
|
133
146
|
def flush(events, final)
|
134
147
|
if @compress == 0
|
135
|
-
|
136
|
-
|
148
|
+
data = events.to_json
|
149
|
+
compress_type = 'none'
|
137
150
|
else
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
151
|
+
gz = StringIO.new("w")
|
152
|
+
gz.set_encoding("BINARY")
|
153
|
+
z = Zlib::GzipWriter.new(gz)
|
154
|
+
z.write(events.to_json)
|
155
|
+
z.close
|
156
|
+
data = gz.string
|
157
|
+
compress_type = 'gzip'
|
158
|
+
end
|
159
|
+
headers = {'appid' => @appid, 'version' => PLUGIN_VERSION, 'user-agent' => 'logstash_' + PLUGIN_VERSION,
|
160
|
+
'compress' => compress_type}
|
161
|
+
until do_send(data, headers)
|
162
|
+
sleep 5
|
145
163
|
end
|
146
|
-
headers ={'appid' => @appid,'version'=>PLUGIN_VERSION,'user-agent' =>'logstash_'+ PLUGIN_VERSION,
|
147
|
-
'compress' => compress_type}
|
148
|
-
until do_send(data,headers)
|
149
|
-
sleep 5
|
150
|
-
end
|
151
164
|
@total_send_count += events.length
|
152
165
|
end
|
153
166
|
|
154
167
|
private
|
168
|
+
|
155
169
|
def do_send(data, headers)
|
156
170
|
begin
|
157
|
-
response = @client.post(@url, :body => data
|
171
|
+
response = @client.post(@url, :body => data, :headers => headers).call
|
158
172
|
if response.code != 200
|
159
|
-
@logger.error("Send failed, code: #{response.code}, body: #{response.body}"
|
160
|
-
|
173
|
+
@logger.error("Send failed, code: #{response.code}, body: #{response.body}", :url => @url)
|
174
|
+
return false
|
161
175
|
end
|
162
176
|
rescue => e
|
163
|
-
@logger.error("Send failed"
|
177
|
+
@logger.error("Send failed", :url => @url, :exception => e.class.name, :backtrace => e.backtrace)
|
164
178
|
return false
|
165
179
|
end
|
166
180
|
true
|
167
181
|
end
|
168
182
|
|
169
183
|
private
|
184
|
+
|
170
185
|
def is_filebeat_input?(event)
|
171
186
|
type = event.get("[agent][type]")
|
172
187
|
return true if !type.nil? && type == "filebeat"
|
@@ -178,6 +193,7 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
178
193
|
end
|
179
194
|
|
180
195
|
private
|
196
|
+
|
181
197
|
def record_filebeat_status(log_detail, offset)
|
182
198
|
status = @filebeat_status[log_detail]
|
183
199
|
if status.nil?
|
@@ -190,15 +206,16 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
190
206
|
end
|
191
207
|
|
192
208
|
private
|
209
|
+
|
193
210
|
def report
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
211
|
+
interval_count = @total_send_count - @last_report_count
|
212
|
+
@last_report_count = @total_send_count
|
213
|
+
@logger.info("Report:",
|
214
|
+
"IntervalReceive(records):" => interval_count,
|
215
|
+
"Receive(records):" => @receive_count,
|
216
|
+
"TotalSend(records):" => @total_send_count,
|
217
|
+
"ParseError(records):" => @parse_error_count)
|
218
|
+
@logger.info("Filebeat Status Report: #{format_filebeat_report}") if @is_filebeat_status_record && @filebeat_status != {}
|
202
219
|
end
|
203
220
|
|
204
221
|
private
|
@@ -212,4 +229,6 @@ class LogStash::Outputs::Thinkingdata < LogStash::Outputs::Base
|
|
212
229
|
result
|
213
230
|
end
|
214
231
|
|
215
|
-
end
|
232
|
+
end
|
233
|
+
|
234
|
+
# class LogStash::Outputs::Thinkingdata
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-thinkingdata'
|
3
|
-
s.version = '1.0.
|
3
|
+
s.version = '1.0.1'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Output plugin for Thinkingdata Analytics'
|
6
6
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-thinkingdata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- sdk
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core-plugin-api
|
@@ -134,6 +134,7 @@ files:
|
|
134
134
|
- Gemfile
|
135
135
|
- LICENSE
|
136
136
|
- README.md
|
137
|
+
- lib/logstash/filter/sensors/sensors_data.rb
|
137
138
|
- lib/logstash/outputs/thinkingdata.rb
|
138
139
|
- logstash-output-thinkingdata.gemspec
|
139
140
|
- spec/outputs/thinkingdata_spec.rb
|
@@ -143,7 +144,7 @@ licenses:
|
|
143
144
|
metadata:
|
144
145
|
logstash_plugin: 'true'
|
145
146
|
logstash_group: output
|
146
|
-
post_install_message:
|
147
|
+
post_install_message:
|
147
148
|
rdoc_options: []
|
148
149
|
require_paths:
|
149
150
|
- lib
|
@@ -158,8 +159,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
158
159
|
- !ruby/object:Gem::Version
|
159
160
|
version: '0'
|
160
161
|
requirements: []
|
161
|
-
rubygems_version: 3.0.
|
162
|
-
signing_key:
|
162
|
+
rubygems_version: 3.0.8
|
163
|
+
signing_key:
|
163
164
|
specification_version: 4
|
164
165
|
summary: Output plugin for Thinkingdata Analytics
|
165
166
|
test_files:
|