logstash-output-sensors_analytics 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 56e114454ed3a71e4c6977bee6166627e1d54b7a3e483f8f54803197dd70038a
4
- data.tar.gz: 4439c667f828bae8d53bbb059924d61f4f98f876fae4847e3c14a21081fe5d8d
3
+ metadata.gz: 04a13a5a7e38d72dba2dc7064a3c567789c7109dc06cc2a7fc5b4fb9d0d0a1eb
4
+ data.tar.gz: 46c542e7c46e32f22d30e5d503b3163b29bea49b18ee5161d1d4a28d027e5357
5
5
  SHA512:
6
- metadata.gz: c59e5b2adf9cf5474e4455d097f2ce97539055229d7a8496b00eb00846077feab303313964893eb8b009a092cde7d7bcf24d03f6242b4d44b3a16ef858fed3bc
7
- data.tar.gz: 5cf2456fb8cdb18d5346162137bb6fdef3a6f9d91405f2adf1d244b35229ee64506c09cad796a776d486293e40275f9d6adc73351e15939c7e209b12a9a4ff12
6
+ metadata.gz: 6c0bdcb072ceb310f92b25e5da47d89d563c56e5d84fcb99229a6e0136a8d3f76cde19062187c1d965ba61d866ab7518273252e33d98547585eb8529c1998c4e
7
+ data.tar.gz: d97909fd5156adc44cdfd2eb35be5479934a9019311095bdcc10378aa7435f39477e51e2d28cea8a10891dc9edd90ed97461d952eff426a559d1d635ce47403a
data/README.md CHANGED
@@ -49,15 +49,19 @@ bundle exec rspec
49
49
 
50
50
  - Edit Logstash `Gemfile` and add the local plugin path, for example:
51
51
  ```ruby
52
- gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
52
+ gem "logstash-output-sensors_analytics", :path => "/your/local/logstash-output-sensors_analytics"
53
53
  ```
54
54
  - Install plugin
55
55
  ```sh
56
+ # Logstash 2.3 and higher
56
57
  bin/logstash-plugin install --no-verify
58
+
59
+ # Prior to Logstash 2.3
60
+ bin/plugin install --no-verify
57
61
  ```
58
62
  - Run Logstash with your plugin
59
63
  ```sh
60
- bin/logstash -e 'filter {awesome {}}'
64
+ bin/logstash -e 'output {sensors_analytics {url => "https://example.sensorsdata.cn:8106/sa"}}'
61
65
  ```
62
66
  At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
63
67
 
@@ -67,11 +71,11 @@ You can use the same **2.1** method to run your plugin in an installed Logstash
67
71
 
68
72
  - Build your plugin gem
69
73
  ```sh
70
- gem build logstash-filter-awesome.gemspec
74
+ gem build logstash-output-sensors_analytics.gemspec
71
75
  ```
72
76
  - Install the plugin from the Logstash home
73
77
  ```sh
74
- bin/logstash-plugin install /your/local/plugin/logstash-filter-awesome.gem
78
+ bin/logstash-plugin install /your/local/plugin/logstash-output-sensors_analytics.gem
75
79
  ```
76
80
  - Start Logstash and proceed to test the plugin
77
81
 
@@ -8,16 +8,14 @@ require "json"
8
8
 
9
9
  # An sensors_analytics output that does nothing.
10
10
  class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
11
- include Stud::Buffer
12
11
  include LogStash::PluginMixins::HttpClient
13
12
 
14
- attr_accessor :buffer_state
15
-
16
13
  config_name "sensors_analytics"
14
+
17
15
  concurrency :single
18
16
 
19
- # URL to use
20
- config :url, :validate => :string, :required => :true
17
+ # 数据接收地址, 可配置多个
18
+ config :url, :validate => :array, :required => :true
21
19
 
22
20
  # 数据的项目
23
21
  config :project, :validate => :string
@@ -28,67 +26,247 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
28
26
  # 批次最大 record 数量
29
27
  config :flush_batch_size, :validate => :number, :default => 100
30
28
 
31
- PLUGIN_VERSION = "0.1.1"
29
+ # 数据中用做 hash 的值
30
+ config :hash_filed, :validate => :array
31
+
32
+ # 开启 filebeat 状态记录
33
+ config :enable_filebeat_status_report, :validate => :boolean, :default => true
34
+
35
+ PLUGIN_VERSION = "0.1.2"
32
36
 
33
37
  public
38
+
34
39
  def register
35
40
  @logger.info("Registering sensors_analytics Output",
41
+ :version => PLUGIN_VERSION,
36
42
  :url => @url,
37
43
  :flush_interval_sec => @flush_interval_sec,
38
- :flush_batch_size => @flush_batch_size
44
+ :flush_batch_size => @flush_batch_size,
45
+ :hash_filed => @hash_filed,
46
+ :enable_filebeat_status_report => @enable_filebeat_status_report
39
47
  )
40
48
 
41
49
  http_client_config = client_config
42
50
  http_client_config[:user_agent] = "SensorsAnalytics Logstash Output Plugin " + PLUGIN_VERSION
43
51
  @client = Manticore::Client.new(http_client_config)
52
+ @buffer_items = []
53
+ @receive_count = 0
54
+ @parse_error_count = 0
55
+ @last_report_time = Time.now
56
+ @last_report_count = 0
57
+ @url.each_index do |i|
58
+ option = {
59
+ :flush_batch_size => @flush_batch_size,
60
+ :flush_interval_sec => @flush_interval_sec,
61
+ :client => @client,
62
+ :url_list => @url,
63
+ :index => i,
64
+ :logger => @logger
65
+ }
66
+ buffer_item = BufferItem.new(option)
67
+ @buffer_items << buffer_item
68
+ end
44
69
 
45
- buffer_config = {
46
- :max_items => @flush_batch_size.to_i,
47
- :max_interval => @flush_interval_sec.to_i,
48
- :logger => @logger
49
- }
70
+ @recent_filebeat_status = {} if @enable_filebeat_status_report
71
+ @report_thread = Thread.new do
72
+ loop do
73
+ sleep 60
74
+ report
75
+ end
76
+ end
50
77
 
51
- buffer_initialize(buffer_config)
52
78
  end
53
79
 
54
80
  public
81
+
55
82
  def multi_receive(events)
56
83
  return if events.empty?
57
-
84
+ @receive_count += events.length
58
85
  events.each do |e|
59
86
  begin
60
87
  record = JSON.parse(e.get("message"))
61
-
62
- host = e.get("host")
63
- host = host["name"] if host != nil and host.is_a?(Hash)
64
-
65
- path = e.get("path")
66
- path = e.get("[log][file][path]") if path == nil
88
+ tag = concat_tag_from_hash_filed(e)
89
+ if filebeat_input?(e)
90
+ host = e.get("[host][name]")
91
+ file = e.get("[log][file][path]")
92
+ offset = e.get("[log][offset]")
93
+ lib_detail = "#{host}###{file}"
94
+ tag = host.to_s + file.to_s if tag.nil?
95
+ collect_filebeat_status(lib_detail, offset) if @enable_filebeat_status_report
96
+ else
97
+ # 这里记录一个 file_input 的 lib_detail, 其他 input 为空
98
+ host = e.get("host")
99
+ path = e.get("path")
100
+ if !host.nil? && !path.nil?
101
+ lib_detail = "#{host}###{path}"
102
+ tag = host.to_s + path.to_s if tag.nil?
103
+ else
104
+ lib_detail = ""
105
+ end
106
+ end
67
107
 
68
108
  record["lib"] = {
69
109
  "$lib" => "Logstash",
70
110
  "$lib_version" => PLUGIN_VERSION,
71
111
  "$lib_method" => "tools",
72
- "$lib_detail" => "#{host}###{path}"
112
+ "$lib_detail" => lib_detail
73
113
  }
74
114
 
75
115
  record["project"] = @project if @project != nil
76
116
 
77
- buffer_receive(record)
117
+ buffer_item = @buffer_items[buffer_index(tag)]
118
+ buffer_item.buffer_receive(record)
78
119
  rescue
79
120
  @logger.error("Could not process record", :record => e.to_s)
121
+ @parse_error_count += 1
80
122
  end
81
123
  end
82
124
  end
83
125
 
84
126
  public
127
+
85
128
  def close
86
- buffer_flush(:final => true)
87
- client.close
129
+ @buffer_items.each do |buffer_item|
130
+ buffer_item.buffer_state[:timer].kill
131
+ buffer_item.buffer_flush(:final => true)
132
+ end
133
+ @report_thread.kill
134
+ @client.close
135
+ report
136
+ end
137
+
138
+ private
139
+
140
+ def buffer_index(tag)
141
+ tag.hash % @url.length
142
+ end
143
+
144
+ private
145
+
146
+ def concat_tag_from_hash_filed(event)
147
+ if !@hash_filed.nil? && !@hash_filed.empty?
148
+ tag = ""
149
+ @hash_filed.each do |filed|
150
+ tag << event.get(filed).to_s
151
+ end
152
+ return tag
153
+ end
154
+ nil
155
+ end
156
+
157
+ private
158
+
159
+ def filebeat_input?(event)
160
+ tag = event.get("[agent][type]")
161
+ return true if !tag.nil? && tag == "filebeat"
162
+ tag = event.get("[@metadata][beat]")
163
+ return true if !tag.nil? && tag == "filebeat"
164
+ false
165
+ end
166
+
167
+ private
168
+
169
+ def collect_filebeat_status(lib_detail, offset)
170
+ status = @recent_filebeat_status[lib_detail]
171
+ if status.nil?
172
+ status = {:receive_time => Time.now, :offset => offset}
173
+ @recent_filebeat_status[lib_detail] = status
174
+ else
175
+ status[:offset] = offset
176
+ status[:receive_time] = Time.now
177
+ end
178
+ end
179
+
180
+ private
181
+
182
+ def format_filebeat_report_and_clean
183
+ result = "\n"
184
+ @recent_filebeat_status.each do |k, v|
185
+ result << k << "=>" << v.to_s << "\n"
186
+ end
187
+ @recent_filebeat_status = {}
188
+ result
189
+ end
190
+
191
+ public
192
+
193
+ def report
194
+ url_send_count_sum = {}
195
+ @url.each do |url|
196
+ url_send_count_sum[url] = 0
197
+ end
198
+
199
+ @buffer_items.each do |buffer_item|
200
+ buffer_url_send_count = buffer_item.url_send_count
201
+ buffer_url_send_count.each do |url, count|
202
+ url_send_count_sum[url] += count
203
+ end
204
+ end
205
+
206
+ total_send_count = 0
207
+ url_send_count_sum.each do |url, count|
208
+ total_send_count += count;
209
+ end
210
+
211
+ speed = (total_send_count - @last_report_count) / (Time.now - @last_report_time)
212
+ @last_report_count = total_send_count
213
+ @last_report_time = Time.now
214
+ @logger.info("Report",
215
+ :speed => speed.round(2),
216
+ :receive_count => @receive_count,
217
+ :send_count => total_send_count,
218
+ :parse_error_count => @parse_error_count,
219
+ :url_send_count => url_send_count_sum)
220
+ @logger.info("Filebeat status Report: #{format_filebeat_report_and_clean}") if @enable_filebeat_status_report
221
+ end
222
+
223
+ end # class LogStash::Outputs::SensorsAnalytics
224
+
225
+
226
+ class BufferItem
227
+ include Stud::Buffer
228
+
229
+ attr_accessor :buffer_state
230
+ attr_accessor :url_send_count
231
+
232
+ def initialize(option = {})
233
+ @client = option[:client]
234
+ @url_send_count = {}
235
+ url_list = option[:url_list]
236
+ url_list.each do |url|
237
+ @url_send_count[url] = 0
238
+ end
239
+ init_url_list(url_list, option[:index])
240
+ @logger = option[:logger]
241
+
242
+ buffer_config = {
243
+ :max_items => option[:flush_batch_size],
244
+ :max_interval => option[:flush_interval_sec],
245
+ :logger => @logger
246
+ }
247
+ buffer_initialize(buffer_config)
248
+ end
249
+
250
+ def do_send(form_data, url)
251
+ begin
252
+ response = @client.post(url, :params => form_data).call
253
+ if response.code != 200
254
+ @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
255
+ return false
256
+ end
257
+ rescue => e
258
+ @logger.warn("Send failed", :exception => e.class.name, :backtrace => e.backtrace)
259
+ return false
260
+ end
261
+ true
88
262
  end
89
263
 
90
264
  public
91
- def flush(events, final = false)
265
+
266
+ # 数据被 Gzip > Base64 后尝试发送
267
+ # 如果当前 url 发送失败, 会尝试获取列表中下一个地址进行发送, 发送失败的 url 在 3 秒内不会再尝试发送
268
+ # 如果所有的 url 都被标记为发送失败, sleep 5 秒后重新获取
269
+ def flush(events, final)
92
270
  wio = StringIO.new("w")
93
271
  gzip_io = Zlib::GzipWriter.new(wio)
94
272
  gzip_io.write(events.to_json)
@@ -96,11 +274,56 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
96
274
  data = Base64.strict_encode64(wio.string)
97
275
  form_data = {"data_list" => data, "gzip" => 1}
98
276
 
99
- response = client.post(@url, :params => form_data).call
100
- if response.code != 200
101
- @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
102
- raise
277
+ url_item = obtain_url
278
+
279
+ until do_send(form_data, url_item[:url])
280
+ last_url = url_item[:url]
281
+ # 将发送失败的 url 标记为不可用
282
+ disable_url(url_item)
283
+ url_item = obtain_url
284
+ @logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
103
285
  end
286
+ @url_send_count[url_item[:url]] += events.length
104
287
  end
105
288
 
106
- end # class LogStash::Outputs::SensorsAnalytics
289
+ private
290
+
291
+ # 把当前 buffer 用的 url 从 list 的 0 索引开始依次放入, 方便在 obtain_url 遍历
292
+ def init_url_list(urls, start_index)
293
+ @url_list = []
294
+ index = start_index
295
+ loop do
296
+ @url_list << {
297
+ :url => urls[index],
298
+ :ok? => true,
299
+ :fail_time => Time.now
300
+ }
301
+ index = (index + 1) % urls.length
302
+ break if index == start_index
303
+ end
304
+ end
305
+
306
+
307
+ private
308
+
309
+ def obtain_url
310
+ while true do
311
+ @url_list.each do |url_item|
312
+ return url_item if url_item[:ok?]
313
+ if Time.now - url_item[:fail_time] > 3
314
+ url_item[:ok] = true
315
+ return url_item
316
+ end
317
+ end
318
+ @logger.warn("All url disable, sleep 5 sec")
319
+ sleep 5
320
+ end
321
+ end
322
+
323
+ private
324
+
325
+ def disable_url(url_item)
326
+ url_item[:ok?] = false
327
+ url_item[:fail_time] = Time.now
328
+ end
329
+ end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-sensors_analytics'
3
- s.version = '0.1.0'
3
+ s.version = '0.1.2'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = 'Output plugin for Sensors Analytics'
6
6
  s.description = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'
@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
3
3
  require "logstash/outputs/sensors_analytics"
4
4
  require "logstash/codecs/plain"
5
5
  require "logstash/event"
6
+ require "base64"
6
7
 
7
8
  PORT = rand(65535 - 1024) + 1025
8
9
 
@@ -19,6 +20,38 @@ class TestApp < Sinatra::Base
19
20
  end
20
21
  end
21
22
 
23
+ def self.receive_count=(count)
24
+ @receive_count = count
25
+ end
26
+
27
+ def self.receive_count
28
+ @receive_count || 0
29
+ end
30
+
31
+ def self.mult_0=(count)
32
+ @mult_0 = count
33
+ end
34
+
35
+ def self.mult_0
36
+ @mult_0 || 0
37
+ end
38
+
39
+ def self.mult_1=(count)
40
+ @mult_1 = count
41
+ end
42
+
43
+ def self.mult_1
44
+ @mult_1 || 0
45
+ end
46
+
47
+ def self.mult_2=(count)
48
+ @mult_2 = count
49
+ end
50
+
51
+ def self.mult_2
52
+ @mult_2 || 0
53
+ end
54
+
22
55
  def self.last_request=(request)
23
56
  @last_request = request
24
57
  end
@@ -53,6 +86,28 @@ class TestApp < Sinatra::Base
53
86
 
54
87
  multiroute(%w(post), "/good") do
55
88
  self.class.last_request = request
89
+ self.class.receive_count += 1
90
+ [200, "YUP"]
91
+ end
92
+
93
+ multiroute(%w(post), "/mult_0") do
94
+ self.class.last_request = request
95
+ self.class.receive_count += 1
96
+ self.class.mult_0 += 1
97
+ [200, "YUP"]
98
+ end
99
+
100
+ multiroute(%w(post), "/mult_1") do
101
+ self.class.last_request = request
102
+ self.class.receive_count += 1
103
+ self.class.mult_1 += 1
104
+ [200, "YUP"]
105
+ end
106
+
107
+ multiroute(%w(post), "/mult_2") do
108
+ self.class.last_request = request
109
+ self.class.receive_count += 1
110
+ self.class.mult_2 += 1
56
111
  [200, "YUP"]
57
112
  end
58
113
 
@@ -105,9 +160,20 @@ end
105
160
 
106
161
  describe LogStash::Outputs::SensorsAnalytics do
107
162
 
163
+ # 解析 request body
164
+ def parse_data(request_data)
165
+ data = request_data[request_data.index("=") + 1...request_data.index("&gzip")]
166
+ url_decode = URI::decode(data)
167
+ base_64 = Base64.strict_decode64(url_decode)
168
+ io = StringIO.new(base_64)
169
+ gzip_io = Zlib::GzipReader.new(io)
170
+ json_str = gzip_io.read
171
+ JSON.parse(json_str)
172
+ end
173
+
108
174
  describe "basic test" do
109
175
 
110
- let(:port) {PORT}
176
+ let(:port) { PORT }
111
177
  let(:event) {
112
178
  event = LogStash::Event.new
113
179
  event.set("path", "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
@@ -115,12 +181,12 @@ describe LogStash::Outputs::SensorsAnalytics do
115
181
  event.set("message", '{"distinct_id":"123456","time":1434556935000,"type":"track","event":"ViewProduct","properties":{"product_id":12345,"product_name":"苹果","product_classify":"水果","product_price":14}}')
116
182
  event
117
183
  }
118
- let(:method) {"post"}
184
+ let(:method) { "post" }
119
185
 
120
186
  context 'sending no events' do
121
- let(:url) {"http://localhost:#{port}/good"}
122
- let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1}}
123
- subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
187
+ let(:url) { "http://localhost:#{port}/good" }
188
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
189
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
124
190
 
125
191
  before do
126
192
  subject.register
@@ -131,39 +197,171 @@ describe LogStash::Outputs::SensorsAnalytics do
131
197
  end
132
198
  end
133
199
 
200
+ context "send events" do
201
+ let(:url) { "http://localhost:#{port}/good" }
202
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 10, "flush_interval_sec" => 2} }
203
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
204
+
205
+ before do
206
+ subject.register
207
+ TestApp.receive_count = 0
208
+ end
209
+
210
+ it 'should send requests 3 times' do
211
+ 30.times { subject.multi_receive([event]) }
212
+ expect(TestApp.receive_count).to eq(3)
213
+ end
214
+
215
+ before do
216
+ TestApp.receive_count = 0
217
+ end
218
+
219
+ it 'should send the request after 2 seconds' do
220
+ 7.times { subject.multi_receive([event]) }
221
+ expect(TestApp.receive_count).to eq(0)
222
+ sleep(3)
223
+ expect(TestApp.receive_count).to eq(1)
224
+ end
225
+ end
226
+
134
227
  context "with retryable failing requests" do
135
- let(:url) {"http://localhost:#{port}/retry"}
136
- let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "flush_interval" => 100}}
137
- subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
228
+ let(:url) { "http://localhost:#{port}/retry" }
229
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "flush_interval_sec" => 100} }
230
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
138
231
 
139
232
  before do
140
233
  subject.register
141
234
  end
142
235
 
143
236
  before do
144
- TestApp.retry_fail_count = 3
237
+ TestApp.retry_fail_count = 5
238
+ event.set("@metadata", {"beat" => "filebeat"})
239
+ event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
240
+ event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
145
241
  subject.multi_receive([event])
146
242
  end
147
243
 
148
244
  it "should log a retryable response 6 times" do
245
+ expect(TestApp.retry_action_count).to eq(6)
149
246
  expect(TestApp.final_success).to eq(true)
150
- expect(TestApp.retry_action_count).to eq(4)
151
- expect(TestApp.last_request.body.read).to eq('data_list=H4sIAAAAAAAA%2F1WPQU7DMBBF7zJlmcZJ2xSRJWuQ2MAGoWqwp%2B20bhzZ06Kqygm4AuIMLNhwoYpjYEe0iJX93x%2F9P%2FN4AMNBuNEyYwM1lKPxpJpCBsIbgrqcRFlNr8ZVURQR7tsIQTzqdZyhHTUS9QPTy513Zqsl0ta7lrwwBagPSSXep%2Ffh2Rk1mCrg%2B%2FXr%2BP4Gf1xbDIHn%2B%2BgdPz7%2Fe61n3e%2FVZWD5OTVc9C%2FcuEUQDMs4nMhsRz6wa6JT5GVenvCGZOnSpeKcDSdqSJBtSsF2K2joFvW1c%2BthPCu3TqMdDNR9iJFqTs1ixbhiUn2Esr%2FF58%2FwMhUqg4IjhdB1Tz8MgoViZwEAAA%3D%3D&gzip=1')
247
+ event_obj = parse_data(TestApp.last_request.body.read)[0]
248
+ expect(event_obj["event"]).to eq("ViewProduct")
249
+ expect(event_obj["distinct_id"]).to eq("123456")
250
+ expect(event_obj["lib"]["$lib_detail"])
251
+ .to eq("LaputadeMacBook-Pro.local##/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
152
252
  end
153
253
  end
154
254
 
155
255
  context 'config project' do
156
- let(:url) {"http://localhost:#{port}/good"}
157
- let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "project" => "production"}}
158
- subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
256
+ let(:url) { "http://localhost:#{port}/good" }
257
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "project" => "production"} }
258
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
159
259
 
160
260
  before do
261
+ event.set("@metadata", {"beat" => "filebeat"})
262
+ event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
263
+ event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
161
264
  subject.register
162
265
  subject.multi_receive([event])
163
266
  end
164
267
 
165
268
  it 'should add project into record' do
166
- expect(TestApp.last_request.body.read).to eq('data_list=H4sIAAAAAAAA%2F1WQTU7DQAyF7%2BKyTDNJ2xSRJWuQ2MAGocpM3NbpNBNl3KKqygm4AuIMLNhwoYpjMDPqj1jN%2BLP13rOf91CxE260zLiCEvLReFJMIQHhNUGZT3xZTG%2FGRZZlHu5aD0E61Cs%2FQ1tqxNdPTG8Pna02WjxtO9tSJ0wOyn2oAo%2FqUTw5owaDBfy%2B%2Fxw%2BP%2BDCtUHneL7zvcPX9%2F9e27GOufoEDL8Gh6v4wp1dOEG39MOBzLbUObaN72RpnuYnvCZZ2rCpWGvciVYkyCaoYLsRrOge9a21q6FfKzVWoxkM1KPzkmpOzaJmrJlUlFDmaHz%2BDK%2BDoapQcKQQ%2Bpi%2BJh1OddwjBOtf%2FgAJu5tJfgEAAA%3D%3D&gzip=1')
269
+ event_obj = parse_data(TestApp.last_request.body.read)[0]
270
+ expect(event_obj["project"]).to eq("production")
271
+ end
272
+ end
273
+
274
+ context 'config multiple url' do
275
+ let(:url) { Array["http://localhost:#{port}/mult_0",
276
+ "http://localhost:#{port}/mult_1",
277
+ "http://localhost:#{port}/mult_2"] }
278
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
279
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
280
+
281
+ # filebeat input 时默认通过 hostname + path 做 hash
282
+ before do
283
+ TestApp.mult_0 = 0
284
+ TestApp.mult_1 = 0
285
+ TestApp.mult_1 = 0
286
+ TestApp.receive_count = 0
287
+ subject.register
288
+ event.set("@metadata", {"beat" => "filebeat"})
289
+ event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
290
+ (0...100).each { |i|
291
+ event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
292
+ subject.multi_receive([event])
293
+ }
294
+ end
295
+
296
+ it 'should send event to multiple url' do
297
+ expect(TestApp.mult_0 > 10).to eq(true)
298
+ expect(TestApp.mult_1 > 10).to eq(true)
299
+ expect(TestApp.mult_2 > 10).to eq(true)
300
+ expect(TestApp.receive_count).to eq(100)
301
+ end
302
+ end
303
+
304
+ context 'config multiple url include bad url' do
305
+ let(:url) { Array["http://localhost:#{port}/mult_0",
306
+ "http://localhost:#{port}/mult_1",
307
+ "http://localhost:#{port}/mult_2",
308
+ "http://localhost:#{port}/bad"] }
309
+ let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
310
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
311
+
312
+ # filebeat input 时默认通过 hostname + path 做 hash
313
+ before do
314
+ TestApp.mult_0 = 0
315
+ TestApp.mult_1 = 0
316
+ TestApp.mult_1 = 0
317
+ TestApp.receive_count = 0
318
+ subject.register
319
+ event.set("@metadata", {"beat" => "filebeat"})
320
+ event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
321
+ (0...100).each { |i|
322
+ event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
323
+ subject.multi_receive([event])
324
+ }
325
+ subject.close
326
+ end
327
+
328
+ it 'should send event to multiple url exclude bad url' do
329
+ expect(TestApp.mult_0 > 15).to eq(true)
330
+ expect(TestApp.mult_1 > 15).to eq(true)
331
+ expect(TestApp.mult_2 > 15).to eq(true)
332
+ expect(TestApp.receive_count).to eq(100)
333
+ end
334
+ end
335
+
336
+ context 'config hash filed and send to multiple url include bad url' do
337
+ let(:url) { Array["http://localhost:#{port}/mult_0",
338
+ "http://localhost:#{port}/mult_1",
339
+ "http://localhost:#{port}/mult_2",
340
+ "http://localhost:#{port}/bad"] }
341
+ let(:verb_behavior_config) { {"url" => url,
342
+ "flush_batch_size" => 1,
343
+ "hash_filed" => Array["name", "[@metadata][path]"]} }
344
+ subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
345
+
346
+ # filebeat input 时默认通过 hostname + path 做 hash
347
+ before do
348
+ TestApp.mult_0 = 0
349
+ TestApp.mult_1 = 0
350
+ TestApp.mult_1 = 0
351
+ TestApp.receive_count = 0
352
+ subject.register
353
+ (0...100).each { |i|
354
+ event.set("name", "name#{i}")
355
+ event.set("@metadata", {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"})
356
+ subject.multi_receive([event])
357
+ }
358
+ end
359
+
360
+ it 'should send event to multiple url exclude bad url' do
361
+ expect(TestApp.mult_0 > 10).to eq(true)
362
+ expect(TestApp.mult_1 > 10).to eq(true)
363
+ expect(TestApp.mult_2 > 10).to eq(true)
364
+ expect(TestApp.receive_count).to eq(100)
167
365
  end
168
366
  end
169
367
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-sensors_analytics
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Feng Jiajie
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-20 00:00:00.000000000 Z
11
+ date: 2020-04-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement