logstash-output-sensors_analytics 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -4
- data/lib/logstash/outputs/sensors_analytics.rb +253 -30
- data/logstash-output-sensors_analytics.gemspec +1 -1
- data/spec/outputs/sensors_analytics_spec.rb +213 -15
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 04a13a5a7e38d72dba2dc7064a3c567789c7109dc06cc2a7fc5b4fb9d0d0a1eb
|
4
|
+
data.tar.gz: 46c542e7c46e32f22d30e5d503b3163b29bea49b18ee5161d1d4a28d027e5357
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6c0bdcb072ceb310f92b25e5da47d89d563c56e5d84fcb99229a6e0136a8d3f76cde19062187c1d965ba61d866ab7518273252e33d98547585eb8529c1998c4e
|
7
|
+
data.tar.gz: d97909fd5156adc44cdfd2eb35be5479934a9019311095bdcc10378aa7435f39477e51e2d28cea8a10891dc9edd90ed97461d952eff426a559d1d635ce47403a
|
data/README.md
CHANGED
@@ -49,15 +49,19 @@ bundle exec rspec
|
|
49
49
|
|
50
50
|
- Edit Logstash `Gemfile` and add the local plugin path, for example:
|
51
51
|
```ruby
|
52
|
-
gem "logstash-
|
52
|
+
gem "logstash-output-sensors_analytics", :path => "/your/local/logstash-output-sensors_analytics"
|
53
53
|
```
|
54
54
|
- Install plugin
|
55
55
|
```sh
|
56
|
+
# Logstash 2.3 and higher
|
56
57
|
bin/logstash-plugin install --no-verify
|
58
|
+
|
59
|
+
# Prior to Logstash 2.3
|
60
|
+
bin/plugin install --no-verify
|
57
61
|
```
|
58
62
|
- Run Logstash with your plugin
|
59
63
|
```sh
|
60
|
-
bin/logstash -e '
|
64
|
+
bin/logstash -e 'output {sensors_analytics {url => "https://example.sensorsdata.cn:8106/sa"}}'
|
61
65
|
```
|
62
66
|
At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
|
63
67
|
|
@@ -67,11 +71,11 @@ You can use the same **2.1** method to run your plugin in an installed Logstash
|
|
67
71
|
|
68
72
|
- Build your plugin gem
|
69
73
|
```sh
|
70
|
-
gem build logstash-
|
74
|
+
gem build logstash-output-sensors_analytics.gemspec
|
71
75
|
```
|
72
76
|
- Install the plugin from the Logstash home
|
73
77
|
```sh
|
74
|
-
bin/logstash-plugin install /your/local/plugin/logstash-
|
78
|
+
bin/logstash-plugin install /your/local/plugin/logstash-output-sensors_analytics.gem
|
75
79
|
```
|
76
80
|
- Start Logstash and proceed to test the plugin
|
77
81
|
|
@@ -8,16 +8,14 @@ require "json"
|
|
8
8
|
|
9
9
|
# An sensors_analytics output that does nothing.
|
10
10
|
class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
11
|
-
include Stud::Buffer
|
12
11
|
include LogStash::PluginMixins::HttpClient
|
13
12
|
|
14
|
-
attr_accessor :buffer_state
|
15
|
-
|
16
13
|
config_name "sensors_analytics"
|
14
|
+
|
17
15
|
concurrency :single
|
18
16
|
|
19
|
-
#
|
20
|
-
config :url, :validate => :
|
17
|
+
# 数据接收地址, 可配置多个
|
18
|
+
config :url, :validate => :array, :required => :true
|
21
19
|
|
22
20
|
# 数据的项目
|
23
21
|
config :project, :validate => :string
|
@@ -28,67 +26,247 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
28
26
|
# 批次最大 record 数量
|
29
27
|
config :flush_batch_size, :validate => :number, :default => 100
|
30
28
|
|
31
|
-
|
29
|
+
# 数据中用做 hash 的值
|
30
|
+
config :hash_filed, :validate => :array
|
31
|
+
|
32
|
+
# 开启 filebeat 状态记录
|
33
|
+
config :enable_filebeat_status_report, :validate => :boolean, :default => true
|
34
|
+
|
35
|
+
PLUGIN_VERSION = "0.1.2"
|
32
36
|
|
33
37
|
public
|
38
|
+
|
34
39
|
def register
|
35
40
|
@logger.info("Registering sensors_analytics Output",
|
41
|
+
:version => PLUGIN_VERSION,
|
36
42
|
:url => @url,
|
37
43
|
:flush_interval_sec => @flush_interval_sec,
|
38
|
-
:flush_batch_size => @flush_batch_size
|
44
|
+
:flush_batch_size => @flush_batch_size,
|
45
|
+
:hash_filed => @hash_filed,
|
46
|
+
:enable_filebeat_status_report => @enable_filebeat_status_report
|
39
47
|
)
|
40
48
|
|
41
49
|
http_client_config = client_config
|
42
50
|
http_client_config[:user_agent] = "SensorsAnalytics Logstash Output Plugin " + PLUGIN_VERSION
|
43
51
|
@client = Manticore::Client.new(http_client_config)
|
52
|
+
@buffer_items = []
|
53
|
+
@receive_count = 0
|
54
|
+
@parse_error_count = 0
|
55
|
+
@last_report_time = Time.now
|
56
|
+
@last_report_count = 0
|
57
|
+
@url.each_index do |i|
|
58
|
+
option = {
|
59
|
+
:flush_batch_size => @flush_batch_size,
|
60
|
+
:flush_interval_sec => @flush_interval_sec,
|
61
|
+
:client => @client,
|
62
|
+
:url_list => @url,
|
63
|
+
:index => i,
|
64
|
+
:logger => @logger
|
65
|
+
}
|
66
|
+
buffer_item = BufferItem.new(option)
|
67
|
+
@buffer_items << buffer_item
|
68
|
+
end
|
44
69
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
70
|
+
@recent_filebeat_status = {} if @enable_filebeat_status_report
|
71
|
+
@report_thread = Thread.new do
|
72
|
+
loop do
|
73
|
+
sleep 60
|
74
|
+
report
|
75
|
+
end
|
76
|
+
end
|
50
77
|
|
51
|
-
buffer_initialize(buffer_config)
|
52
78
|
end
|
53
79
|
|
54
80
|
public
|
81
|
+
|
55
82
|
def multi_receive(events)
|
56
83
|
return if events.empty?
|
57
|
-
|
84
|
+
@receive_count += events.length
|
58
85
|
events.each do |e|
|
59
86
|
begin
|
60
87
|
record = JSON.parse(e.get("message"))
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
88
|
+
tag = concat_tag_from_hash_filed(e)
|
89
|
+
if filebeat_input?(e)
|
90
|
+
host = e.get("[host][name]")
|
91
|
+
file = e.get("[log][file][path]")
|
92
|
+
offset = e.get("[log][offset]")
|
93
|
+
lib_detail = "#{host}###{file}"
|
94
|
+
tag = host.to_s + file.to_s if tag.nil?
|
95
|
+
collect_filebeat_status(lib_detail, offset) if @enable_filebeat_status_report
|
96
|
+
else
|
97
|
+
# 这里记录一个 file_input 的 lib_detail, 其他 input 为空
|
98
|
+
host = e.get("host")
|
99
|
+
path = e.get("path")
|
100
|
+
if !host.nil? && !path.nil?
|
101
|
+
lib_detail = "#{host}###{path}"
|
102
|
+
tag = host.to_s + path.to_s if tag.nil?
|
103
|
+
else
|
104
|
+
lib_detail = ""
|
105
|
+
end
|
106
|
+
end
|
67
107
|
|
68
108
|
record["lib"] = {
|
69
109
|
"$lib" => "Logstash",
|
70
110
|
"$lib_version" => PLUGIN_VERSION,
|
71
111
|
"$lib_method" => "tools",
|
72
|
-
"$lib_detail" =>
|
112
|
+
"$lib_detail" => lib_detail
|
73
113
|
}
|
74
114
|
|
75
115
|
record["project"] = @project if @project != nil
|
76
116
|
|
77
|
-
|
117
|
+
buffer_item = @buffer_items[buffer_index(tag)]
|
118
|
+
buffer_item.buffer_receive(record)
|
78
119
|
rescue
|
79
120
|
@logger.error("Could not process record", :record => e.to_s)
|
121
|
+
@parse_error_count += 1
|
80
122
|
end
|
81
123
|
end
|
82
124
|
end
|
83
125
|
|
84
126
|
public
|
127
|
+
|
85
128
|
def close
|
86
|
-
|
87
|
-
|
129
|
+
@buffer_items.each do |buffer_item|
|
130
|
+
buffer_item.buffer_state[:timer].kill
|
131
|
+
buffer_item.buffer_flush(:final => true)
|
132
|
+
end
|
133
|
+
@report_thread.kill
|
134
|
+
@client.close
|
135
|
+
report
|
136
|
+
end
|
137
|
+
|
138
|
+
private
|
139
|
+
|
140
|
+
def buffer_index(tag)
|
141
|
+
tag.hash % @url.length
|
142
|
+
end
|
143
|
+
|
144
|
+
private
|
145
|
+
|
146
|
+
def concat_tag_from_hash_filed(event)
|
147
|
+
if !@hash_filed.nil? && !@hash_filed.empty?
|
148
|
+
tag = ""
|
149
|
+
@hash_filed.each do |filed|
|
150
|
+
tag << event.get(filed).to_s
|
151
|
+
end
|
152
|
+
return tag
|
153
|
+
end
|
154
|
+
nil
|
155
|
+
end
|
156
|
+
|
157
|
+
private
|
158
|
+
|
159
|
+
def filebeat_input?(event)
|
160
|
+
tag = event.get("[agent][type]")
|
161
|
+
return true if !tag.nil? && tag == "filebeat"
|
162
|
+
tag = event.get("[@metadata][beat]")
|
163
|
+
return true if !tag.nil? && tag == "filebeat"
|
164
|
+
false
|
165
|
+
end
|
166
|
+
|
167
|
+
private
|
168
|
+
|
169
|
+
def collect_filebeat_status(lib_detail, offset)
|
170
|
+
status = @recent_filebeat_status[lib_detail]
|
171
|
+
if status.nil?
|
172
|
+
status = {:receive_time => Time.now, :offset => offset}
|
173
|
+
@recent_filebeat_status[lib_detail] = status
|
174
|
+
else
|
175
|
+
status[:offset] = offset
|
176
|
+
status[:receive_time] = Time.now
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
private
|
181
|
+
|
182
|
+
def format_filebeat_report_and_clean
|
183
|
+
result = "\n"
|
184
|
+
@recent_filebeat_status.each do |k, v|
|
185
|
+
result << k << "=>" << v.to_s << "\n"
|
186
|
+
end
|
187
|
+
@recent_filebeat_status = {}
|
188
|
+
result
|
189
|
+
end
|
190
|
+
|
191
|
+
public
|
192
|
+
|
193
|
+
def report
|
194
|
+
url_send_count_sum = {}
|
195
|
+
@url.each do |url|
|
196
|
+
url_send_count_sum[url] = 0
|
197
|
+
end
|
198
|
+
|
199
|
+
@buffer_items.each do |buffer_item|
|
200
|
+
buffer_url_send_count = buffer_item.url_send_count
|
201
|
+
buffer_url_send_count.each do |url, count|
|
202
|
+
url_send_count_sum[url] += count
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
total_send_count = 0
|
207
|
+
url_send_count_sum.each do |url, count|
|
208
|
+
total_send_count += count;
|
209
|
+
end
|
210
|
+
|
211
|
+
speed = (total_send_count - @last_report_count) / (Time.now - @last_report_time)
|
212
|
+
@last_report_count = total_send_count
|
213
|
+
@last_report_time = Time.now
|
214
|
+
@logger.info("Report",
|
215
|
+
:speed => speed.round(2),
|
216
|
+
:receive_count => @receive_count,
|
217
|
+
:send_count => total_send_count,
|
218
|
+
:parse_error_count => @parse_error_count,
|
219
|
+
:url_send_count => url_send_count_sum)
|
220
|
+
@logger.info("Filebeat status Report: #{format_filebeat_report_and_clean}") if @enable_filebeat_status_report
|
221
|
+
end
|
222
|
+
|
223
|
+
end # class LogStash::Outputs::SensorsAnalytics
|
224
|
+
|
225
|
+
|
226
|
+
class BufferItem
|
227
|
+
include Stud::Buffer
|
228
|
+
|
229
|
+
attr_accessor :buffer_state
|
230
|
+
attr_accessor :url_send_count
|
231
|
+
|
232
|
+
def initialize(option = {})
|
233
|
+
@client = option[:client]
|
234
|
+
@url_send_count = {}
|
235
|
+
url_list = option[:url_list]
|
236
|
+
url_list.each do |url|
|
237
|
+
@url_send_count[url] = 0
|
238
|
+
end
|
239
|
+
init_url_list(url_list, option[:index])
|
240
|
+
@logger = option[:logger]
|
241
|
+
|
242
|
+
buffer_config = {
|
243
|
+
:max_items => option[:flush_batch_size],
|
244
|
+
:max_interval => option[:flush_interval_sec],
|
245
|
+
:logger => @logger
|
246
|
+
}
|
247
|
+
buffer_initialize(buffer_config)
|
248
|
+
end
|
249
|
+
|
250
|
+
def do_send(form_data, url)
|
251
|
+
begin
|
252
|
+
response = @client.post(url, :params => form_data).call
|
253
|
+
if response.code != 200
|
254
|
+
@logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
|
255
|
+
return false
|
256
|
+
end
|
257
|
+
rescue => e
|
258
|
+
@logger.warn("Send failed", :exception => e.class.name, :backtrace => e.backtrace)
|
259
|
+
return false
|
260
|
+
end
|
261
|
+
true
|
88
262
|
end
|
89
263
|
|
90
264
|
public
|
91
|
-
|
265
|
+
|
266
|
+
# 数据被 Gzip > Base64 后尝试发送
|
267
|
+
# 如果当前 url 发送失败, 会尝试获取列表中下一个地址进行发送, 发送失败的 url 在 3 秒内不会再尝试发送
|
268
|
+
# 如果所有的 url 都被标记为发送失败, sleep 5 秒后重新获取
|
269
|
+
def flush(events, final)
|
92
270
|
wio = StringIO.new("w")
|
93
271
|
gzip_io = Zlib::GzipWriter.new(wio)
|
94
272
|
gzip_io.write(events.to_json)
|
@@ -96,11 +274,56 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
|
|
96
274
|
data = Base64.strict_encode64(wio.string)
|
97
275
|
form_data = {"data_list" => data, "gzip" => 1}
|
98
276
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
277
|
+
url_item = obtain_url
|
278
|
+
|
279
|
+
until do_send(form_data, url_item[:url])
|
280
|
+
last_url = url_item[:url]
|
281
|
+
# 将发送失败的 url 标记为不可用
|
282
|
+
disable_url(url_item)
|
283
|
+
url_item = obtain_url
|
284
|
+
@logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
|
103
285
|
end
|
286
|
+
@url_send_count[url_item[:url]] += events.length
|
104
287
|
end
|
105
288
|
|
106
|
-
|
289
|
+
private
|
290
|
+
|
291
|
+
# 把当前 buffer 用的 url 从 list 的 0 索引开始依次放入, 方便在 obtain_url 遍历
|
292
|
+
def init_url_list(urls, start_index)
|
293
|
+
@url_list = []
|
294
|
+
index = start_index
|
295
|
+
loop do
|
296
|
+
@url_list << {
|
297
|
+
:url => urls[index],
|
298
|
+
:ok? => true,
|
299
|
+
:fail_time => Time.now
|
300
|
+
}
|
301
|
+
index = (index + 1) % urls.length
|
302
|
+
break if index == start_index
|
303
|
+
end
|
304
|
+
end
|
305
|
+
|
306
|
+
|
307
|
+
private
|
308
|
+
|
309
|
+
def obtain_url
|
310
|
+
while true do
|
311
|
+
@url_list.each do |url_item|
|
312
|
+
return url_item if url_item[:ok?]
|
313
|
+
if Time.now - url_item[:fail_time] > 3
|
314
|
+
url_item[:ok] = true
|
315
|
+
return url_item
|
316
|
+
end
|
317
|
+
end
|
318
|
+
@logger.warn("All url disable, sleep 5 sec")
|
319
|
+
sleep 5
|
320
|
+
end
|
321
|
+
end
|
322
|
+
|
323
|
+
private
|
324
|
+
|
325
|
+
def disable_url(url_item)
|
326
|
+
url_item[:ok?] = false
|
327
|
+
url_item[:fail_time] = Time.now
|
328
|
+
end
|
329
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-sensors_analytics'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.2'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = 'Output plugin for Sensors Analytics'
|
6
6
|
s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
|
@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
|
|
3
3
|
require "logstash/outputs/sensors_analytics"
|
4
4
|
require "logstash/codecs/plain"
|
5
5
|
require "logstash/event"
|
6
|
+
require "base64"
|
6
7
|
|
7
8
|
PORT = rand(65535 - 1024) + 1025
|
8
9
|
|
@@ -19,6 +20,38 @@ class TestApp < Sinatra::Base
|
|
19
20
|
end
|
20
21
|
end
|
21
22
|
|
23
|
+
def self.receive_count=(count)
|
24
|
+
@receive_count = count
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.receive_count
|
28
|
+
@receive_count || 0
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.mult_0=(count)
|
32
|
+
@mult_0 = count
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.mult_0
|
36
|
+
@mult_0 || 0
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.mult_1=(count)
|
40
|
+
@mult_1 = count
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.mult_1
|
44
|
+
@mult_1 || 0
|
45
|
+
end
|
46
|
+
|
47
|
+
def self.mult_2=(count)
|
48
|
+
@mult_2 = count
|
49
|
+
end
|
50
|
+
|
51
|
+
def self.mult_2
|
52
|
+
@mult_2 || 0
|
53
|
+
end
|
54
|
+
|
22
55
|
def self.last_request=(request)
|
23
56
|
@last_request = request
|
24
57
|
end
|
@@ -53,6 +86,28 @@ class TestApp < Sinatra::Base
|
|
53
86
|
|
54
87
|
multiroute(%w(post), "/good") do
|
55
88
|
self.class.last_request = request
|
89
|
+
self.class.receive_count += 1
|
90
|
+
[200, "YUP"]
|
91
|
+
end
|
92
|
+
|
93
|
+
multiroute(%w(post), "/mult_0") do
|
94
|
+
self.class.last_request = request
|
95
|
+
self.class.receive_count += 1
|
96
|
+
self.class.mult_0 += 1
|
97
|
+
[200, "YUP"]
|
98
|
+
end
|
99
|
+
|
100
|
+
multiroute(%w(post), "/mult_1") do
|
101
|
+
self.class.last_request = request
|
102
|
+
self.class.receive_count += 1
|
103
|
+
self.class.mult_1 += 1
|
104
|
+
[200, "YUP"]
|
105
|
+
end
|
106
|
+
|
107
|
+
multiroute(%w(post), "/mult_2") do
|
108
|
+
self.class.last_request = request
|
109
|
+
self.class.receive_count += 1
|
110
|
+
self.class.mult_2 += 1
|
56
111
|
[200, "YUP"]
|
57
112
|
end
|
58
113
|
|
@@ -105,9 +160,20 @@ end
|
|
105
160
|
|
106
161
|
describe LogStash::Outputs::SensorsAnalytics do
|
107
162
|
|
163
|
+
# 解析 request body
|
164
|
+
def parse_data(request_data)
|
165
|
+
data = request_data[request_data.index("=") + 1...request_data.index("&gzip")]
|
166
|
+
url_decode = URI::decode(data)
|
167
|
+
base_64 = Base64.strict_decode64(url_decode)
|
168
|
+
io = StringIO.new(base_64)
|
169
|
+
gzip_io = Zlib::GzipReader.new(io)
|
170
|
+
json_str = gzip_io.read
|
171
|
+
JSON.parse(json_str)
|
172
|
+
end
|
173
|
+
|
108
174
|
describe "basic test" do
|
109
175
|
|
110
|
-
let(:port) {PORT}
|
176
|
+
let(:port) { PORT }
|
111
177
|
let(:event) {
|
112
178
|
event = LogStash::Event.new
|
113
179
|
event.set("path", "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
|
@@ -115,12 +181,12 @@ describe LogStash::Outputs::SensorsAnalytics do
|
|
115
181
|
event.set("message", '{"distinct_id":"123456","time":1434556935000,"type":"track","event":"ViewProduct","properties":{"product_id":12345,"product_name":"苹果","product_classify":"水果","product_price":14}}')
|
116
182
|
event
|
117
183
|
}
|
118
|
-
let(:method) {"post"}
|
184
|
+
let(:method) { "post" }
|
119
185
|
|
120
186
|
context 'sending no events' do
|
121
|
-
let(:url) {"http://localhost:#{port}/good"}
|
122
|
-
let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1}}
|
123
|
-
subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
|
187
|
+
let(:url) { "http://localhost:#{port}/good" }
|
188
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
|
189
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
124
190
|
|
125
191
|
before do
|
126
192
|
subject.register
|
@@ -131,39 +197,171 @@ describe LogStash::Outputs::SensorsAnalytics do
|
|
131
197
|
end
|
132
198
|
end
|
133
199
|
|
200
|
+
context "send events" do
|
201
|
+
let(:url) { "http://localhost:#{port}/good" }
|
202
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 10, "flush_interval_sec" => 2} }
|
203
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
204
|
+
|
205
|
+
before do
|
206
|
+
subject.register
|
207
|
+
TestApp.receive_count = 0
|
208
|
+
end
|
209
|
+
|
210
|
+
it 'should send requests 3 times' do
|
211
|
+
30.times { subject.multi_receive([event]) }
|
212
|
+
expect(TestApp.receive_count).to eq(3)
|
213
|
+
end
|
214
|
+
|
215
|
+
before do
|
216
|
+
TestApp.receive_count = 0
|
217
|
+
end
|
218
|
+
|
219
|
+
it 'should send the request after 2 seconds' do
|
220
|
+
7.times { subject.multi_receive([event]) }
|
221
|
+
expect(TestApp.receive_count).to eq(0)
|
222
|
+
sleep(3)
|
223
|
+
expect(TestApp.receive_count).to eq(1)
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
134
227
|
context "with retryable failing requests" do
|
135
|
-
let(:url) {"http://localhost:#{port}/retry"}
|
136
|
-
let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "
|
137
|
-
subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
|
228
|
+
let(:url) { "http://localhost:#{port}/retry" }
|
229
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "flush_interval_sec" => 100} }
|
230
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
138
231
|
|
139
232
|
before do
|
140
233
|
subject.register
|
141
234
|
end
|
142
235
|
|
143
236
|
before do
|
144
|
-
TestApp.retry_fail_count =
|
237
|
+
TestApp.retry_fail_count = 5
|
238
|
+
event.set("@metadata", {"beat" => "filebeat"})
|
239
|
+
event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
|
240
|
+
event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
|
145
241
|
subject.multi_receive([event])
|
146
242
|
end
|
147
243
|
|
148
244
|
it "should log a retryable response 6 times" do
|
245
|
+
expect(TestApp.retry_action_count).to eq(6)
|
149
246
|
expect(TestApp.final_success).to eq(true)
|
150
|
-
|
151
|
-
expect(
|
247
|
+
event_obj = parse_data(TestApp.last_request.body.read)[0]
|
248
|
+
expect(event_obj["event"]).to eq("ViewProduct")
|
249
|
+
expect(event_obj["distinct_id"]).to eq("123456")
|
250
|
+
expect(event_obj["lib"]["$lib_detail"])
|
251
|
+
.to eq("LaputadeMacBook-Pro.local##/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
|
152
252
|
end
|
153
253
|
end
|
154
254
|
|
155
255
|
context 'config project' do
|
156
|
-
let(:url) {"http://localhost:#{port}/good"}
|
157
|
-
let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "project" => "production"}}
|
158
|
-
subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
|
256
|
+
let(:url) { "http://localhost:#{port}/good" }
|
257
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "project" => "production"} }
|
258
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
159
259
|
|
160
260
|
before do
|
261
|
+
event.set("@metadata", {"beat" => "filebeat"})
|
262
|
+
event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
|
263
|
+
event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
|
161
264
|
subject.register
|
162
265
|
subject.multi_receive([event])
|
163
266
|
end
|
164
267
|
|
165
268
|
it 'should add project into record' do
|
166
|
-
|
269
|
+
event_obj = parse_data(TestApp.last_request.body.read)[0]
|
270
|
+
expect(event_obj["project"]).to eq("production")
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
context 'config multiple url' do
|
275
|
+
let(:url) { Array["http://localhost:#{port}/mult_0",
|
276
|
+
"http://localhost:#{port}/mult_1",
|
277
|
+
"http://localhost:#{port}/mult_2"] }
|
278
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
|
279
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
280
|
+
|
281
|
+
# filebeat input 时默认通过 hostname + path 做 hash
|
282
|
+
before do
|
283
|
+
TestApp.mult_0 = 0
|
284
|
+
TestApp.mult_1 = 0
|
285
|
+
TestApp.mult_1 = 0
|
286
|
+
TestApp.receive_count = 0
|
287
|
+
subject.register
|
288
|
+
event.set("@metadata", {"beat" => "filebeat"})
|
289
|
+
event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
|
290
|
+
(0...100).each { |i|
|
291
|
+
event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
|
292
|
+
subject.multi_receive([event])
|
293
|
+
}
|
294
|
+
end
|
295
|
+
|
296
|
+
it 'should send event to multiple url' do
|
297
|
+
expect(TestApp.mult_0 > 10).to eq(true)
|
298
|
+
expect(TestApp.mult_1 > 10).to eq(true)
|
299
|
+
expect(TestApp.mult_2 > 10).to eq(true)
|
300
|
+
expect(TestApp.receive_count).to eq(100)
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
304
|
+
context 'config multiple url include bad url' do
|
305
|
+
let(:url) { Array["http://localhost:#{port}/mult_0",
|
306
|
+
"http://localhost:#{port}/mult_1",
|
307
|
+
"http://localhost:#{port}/mult_2",
|
308
|
+
"http://localhost:#{port}/bad"] }
|
309
|
+
let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
|
310
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
311
|
+
|
312
|
+
# filebeat input 时默认通过 hostname + path 做 hash
|
313
|
+
before do
|
314
|
+
TestApp.mult_0 = 0
|
315
|
+
TestApp.mult_1 = 0
|
316
|
+
TestApp.mult_1 = 0
|
317
|
+
TestApp.receive_count = 0
|
318
|
+
subject.register
|
319
|
+
event.set("@metadata", {"beat" => "filebeat"})
|
320
|
+
event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
|
321
|
+
(0...100).each { |i|
|
322
|
+
event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
|
323
|
+
subject.multi_receive([event])
|
324
|
+
}
|
325
|
+
subject.close
|
326
|
+
end
|
327
|
+
|
328
|
+
it 'should send event to multiple url exclude bad url' do
|
329
|
+
expect(TestApp.mult_0 > 15).to eq(true)
|
330
|
+
expect(TestApp.mult_1 > 15).to eq(true)
|
331
|
+
expect(TestApp.mult_2 > 15).to eq(true)
|
332
|
+
expect(TestApp.receive_count).to eq(100)
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
context 'config hash filed and send to multiple url include bad url' do
|
337
|
+
let(:url) { Array["http://localhost:#{port}/mult_0",
|
338
|
+
"http://localhost:#{port}/mult_1",
|
339
|
+
"http://localhost:#{port}/mult_2",
|
340
|
+
"http://localhost:#{port}/bad"] }
|
341
|
+
let(:verb_behavior_config) { {"url" => url,
|
342
|
+
"flush_batch_size" => 1,
|
343
|
+
"hash_filed" => Array["name", "[@metadata][path]"]} }
|
344
|
+
subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
|
345
|
+
|
346
|
+
# filebeat input 时默认通过 hostname + path 做 hash
|
347
|
+
before do
|
348
|
+
TestApp.mult_0 = 0
|
349
|
+
TestApp.mult_1 = 0
|
350
|
+
TestApp.mult_1 = 0
|
351
|
+
TestApp.receive_count = 0
|
352
|
+
subject.register
|
353
|
+
(0...100).each { |i|
|
354
|
+
event.set("name", "name#{i}")
|
355
|
+
event.set("@metadata", {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"})
|
356
|
+
subject.multi_receive([event])
|
357
|
+
}
|
358
|
+
end
|
359
|
+
|
360
|
+
it 'should send event to multiple url exclude bad url' do
|
361
|
+
expect(TestApp.mult_0 > 10).to eq(true)
|
362
|
+
expect(TestApp.mult_1 > 10).to eq(true)
|
363
|
+
expect(TestApp.mult_2 > 10).to eq(true)
|
364
|
+
expect(TestApp.receive_count).to eq(100)
|
167
365
|
end
|
168
366
|
end
|
169
367
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-sensors_analytics
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Feng Jiajie
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|