logstash-output-sensors_analytics 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56e114454ed3a71e4c6977bee6166627e1d54b7a3e483f8f54803197dd70038a
-  data.tar.gz: 4439c667f828bae8d53bbb059924d61f4f98f876fae4847e3c14a21081fe5d8d
+  metadata.gz: 04a13a5a7e38d72dba2dc7064a3c567789c7109dc06cc2a7fc5b4fb9d0d0a1eb
+  data.tar.gz: 46c542e7c46e32f22d30e5d503b3163b29bea49b18ee5161d1d4a28d027e5357
 SHA512:
-  metadata.gz: c59e5b2adf9cf5474e4455d097f2ce97539055229d7a8496b00eb00846077feab303313964893eb8b009a092cde7d7bcf24d03f6242b4d44b3a16ef858fed3bc
-  data.tar.gz: 5cf2456fb8cdb18d5346162137bb6fdef3a6f9d91405f2adf1d244b35229ee64506c09cad796a776d486293e40275f9d6adc73351e15939c7e209b12a9a4ff12
+  metadata.gz: 6c0bdcb072ceb310f92b25e5da47d89d563c56e5d84fcb99229a6e0136a8d3f76cde19062187c1d965ba61d866ab7518273252e33d98547585eb8529c1998c4e
+  data.tar.gz: d97909fd5156adc44cdfd2eb35be5479934a9019311095bdcc10378aa7435f39477e51e2d28cea8a10891dc9edd90ed97461d952eff426a559d1d635ce47403a

data/README.md

@@ -49,15 +49,19 @@ bundle exec rspec
 
 - Edit Logstash `Gemfile` and add the local plugin path, for example:
 ```ruby
-gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+gem "logstash-output-sensors_analytics", :path => "/your/local/logstash-output-sensors_analytics"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
 bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
 ```
 - Run Logstash with your plugin
 ```sh
-bin/logstash -e 'filter {awesome {}}'
+bin/logstash -e 'output {sensors_analytics {url => "https://example.sensorsdata.cn:8106/sa"}}'
 ```
 At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
 
@@ -67,11 +71,11 @@ You can use the same **2.1** method to run your plugin in an installed Logstash
 
 - Build your plugin gem
 ```sh
-gem build logstash-filter-awesome.gemspec
+gem build logstash-output-sensors_analytics.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/logstash-plugin install /your/local/plugin/logstash-filter-awesome.gem
+bin/logstash-plugin install /your/local/plugin/logstash-output-sensors_analytics.gem
 ```
 - Start Logstash and proceed to test the plugin
 

data/lib/logstash/outputs/sensors_analytics.rb

@@ -8,16 +8,14 @@ require "json"
 
 # An sensors_analytics output that does nothing.
 class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
-  include Stud::Buffer
   include LogStash::PluginMixins::HttpClient
 
-  attr_accessor :buffer_state
-
   config_name "sensors_analytics"
+
   concurrency :single
 
-  # URL to use
-  config :url, :validate => :string, :required => :true
+  # 数据接收地址, 可配置多个
+  config :url, :validate => :array, :required => :true
 
   # 数据的项目
   config :project, :validate => :string
@@ -28,67 +26,247 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
   # 批次最大 record 数量
   config :flush_batch_size, :validate => :number, :default => 100
 
-  PLUGIN_VERSION = "0.1.1"
+  # 数据中用做 hash 的值
+  config :hash_filed, :validate => :array
+
+  # 开启 filebeat 状态记录
+  config :enable_filebeat_status_report, :validate => :boolean, :default => true
+
+  PLUGIN_VERSION = "0.1.2"
 
   public
+
   def register
     @logger.info("Registering sensors_analytics Output",
+                 :version => PLUGIN_VERSION,
                  :url => @url,
                  :flush_interval_sec => @flush_interval_sec,
-                 :flush_batch_size => @flush_batch_size
+                 :flush_batch_size => @flush_batch_size,
+                 :hash_filed => @hash_filed,
+                 :enable_filebeat_status_report => @enable_filebeat_status_report
     )
 
     http_client_config = client_config
     http_client_config[:user_agent] = "SensorsAnalytics Logstash Output Plugin " + PLUGIN_VERSION
     @client = Manticore::Client.new(http_client_config)
+    @buffer_items = []
+    @receive_count = 0
+    @parse_error_count = 0
+    @last_report_time = Time.now
+    @last_report_count = 0
+    @url.each_index do |i|
+      option = {
+          :flush_batch_size => @flush_batch_size,
+          :flush_interval_sec => @flush_interval_sec,
+          :client => @client,
+          :url_list => @url,
+          :index => i,
+          :logger => @logger
+      }
+      buffer_item = BufferItem.new(option)
+      @buffer_items << buffer_item
+    end
 
-    buffer_config = {
-        :max_items => @flush_batch_size.to_i,
-        :max_interval => @flush_interval_sec.to_i,
-        :logger => @logger
-    }
+    @recent_filebeat_status = {} if @enable_filebeat_status_report
+    @report_thread = Thread.new do
+      loop do
+        sleep 60
+        report
+      end
+    end
 
-    buffer_initialize(buffer_config)
   end
 
   public
+
   def multi_receive(events)
     return if events.empty?
-
+    @receive_count += events.length
     events.each do |e|
       begin
         record = JSON.parse(e.get("message"))
-
-        host = e.get("host")
-        host = host["name"] if host != nil and host.is_a?(Hash)
-
-        path = e.get("path")
-        path = e.get("[log][file][path]") if path == nil
+        tag = concat_tag_from_hash_filed(e)
+        if filebeat_input?(e)
+          host = e.get("[host][name]")
+          file = e.get("[log][file][path]")
+          offset = e.get("[log][offset]")
+          lib_detail = "#{host}###{file}"
+          tag = host.to_s + file.to_s if tag.nil?
+          collect_filebeat_status(lib_detail, offset) if @enable_filebeat_status_report
+        else
+          # 这里记录一个 file_input 的 lib_detail, 其他 input 为空
+          host = e.get("host")
+          path = e.get("path")
+          if !host.nil? && !path.nil?
+            lib_detail = "#{host}###{path}"
+            tag = host.to_s + path.to_s if tag.nil?
+          else
+            lib_detail = ""
+          end
+        end
 
         record["lib"] = {
             "$lib" => "Logstash",
             "$lib_version" => PLUGIN_VERSION,
             "$lib_method" => "tools",
-            "$lib_detail" => "#{host}###{path}"
+            "$lib_detail" => lib_detail
         }
 
         record["project"] = @project if @project != nil
 
-        buffer_receive(record)
+        buffer_item = @buffer_items[buffer_index(tag)]
+        buffer_item.buffer_receive(record)
       rescue
         @logger.error("Could not process record", :record => e.to_s)
+        @parse_error_count += 1
       end
     end
   end
 
   public
+
   def close
-    buffer_flush(:final => true)
-    client.close
+    @buffer_items.each do |buffer_item|
+      buffer_item.buffer_state[:timer].kill
+      buffer_item.buffer_flush(:final => true)
+    end
+    @report_thread.kill
+    @client.close
+    report
+  end
+
+  private
+
+  def buffer_index(tag)
+    tag.hash % @url.length
+  end
+
+  private
+
+  def concat_tag_from_hash_filed(event)
+    if !@hash_filed.nil? && !@hash_filed.empty?
+      tag = ""
+      @hash_filed.each do |filed|
+        tag << event.get(filed).to_s
+      end
+      return tag
+    end
+    nil
+  end
+
+  private
+
+  def filebeat_input?(event)
+    tag = event.get("[agent][type]")
+    return true if !tag.nil? && tag == "filebeat"
+    tag = event.get("[@metadata][beat]")
+    return true if !tag.nil? && tag == "filebeat"
+    false
+  end
+
+  private
+
+  def collect_filebeat_status(lib_detail, offset)
+    status = @recent_filebeat_status[lib_detail]
+    if status.nil?
+      status = {:receive_time => Time.now, :offset => offset}
+      @recent_filebeat_status[lib_detail] = status
+    else
+      status[:offset] = offset
+      status[:receive_time] = Time.now
+    end
+  end
+
+  private
+
+  def format_filebeat_report_and_clean
+    result = "\n"
+    @recent_filebeat_status.each do |k, v|
+      result << k << "=>" << v.to_s << "\n"
+    end
+    @recent_filebeat_status = {}
+    result
+  end
+
+  public
+
+  def report
+    url_send_count_sum = {}
+    @url.each do |url|
+      url_send_count_sum[url] = 0
+    end
+
+    @buffer_items.each do |buffer_item|
+      buffer_url_send_count = buffer_item.url_send_count
+      buffer_url_send_count.each do |url, count|
+        url_send_count_sum[url] += count
+      end
+    end
+
+    total_send_count = 0
+    url_send_count_sum.each do |url, count|
+      total_send_count += count;
+    end
+
+    speed = (total_send_count - @last_report_count) / (Time.now - @last_report_time)
+    @last_report_count = total_send_count
+    @last_report_time = Time.now
+    @logger.info("Report",
+                 :speed => speed.round(2),
+                 :receive_count => @receive_count,
+                 :send_count => total_send_count,
+                 :parse_error_count => @parse_error_count,
+                 :url_send_count => url_send_count_sum)
+    @logger.info("Filebeat status Report: #{format_filebeat_report_and_clean}") if @enable_filebeat_status_report
+  end
+
+end # class LogStash::Outputs::SensorsAnalytics
+
+
+class BufferItem
+  include Stud::Buffer
+
+  attr_accessor :buffer_state
+  attr_accessor :url_send_count
+
+  def initialize(option = {})
+    @client = option[:client]
+    @url_send_count = {}
+    url_list = option[:url_list]
+    url_list.each do |url|
+      @url_send_count[url] = 0
+    end
+    init_url_list(url_list, option[:index])
+    @logger = option[:logger]
+
+    buffer_config = {
+        :max_items => option[:flush_batch_size],
+        :max_interval => option[:flush_interval_sec],
+        :logger => @logger
+    }
+    buffer_initialize(buffer_config)
+  end
+
+  def do_send(form_data, url)
+    begin
+      response = @client.post(url, :params => form_data).call
+      if response.code != 200
+        @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
+        return false
+      end
+    rescue => e
+      @logger.warn("Send failed", :exception => e.class.name, :backtrace => e.backtrace)
+      return false
+    end
+    true
   end
 
   public
-  def flush(events, final = false)
+
+  # 数据被 Gzip > Base64 后尝试发送
+  # 如果当前 url 发送失败, 会尝试获取列表中下一个地址进行发送, 发送失败的 url 在 3 秒内不会再尝试发送
+  # 如果所有的 url 都被标记为发送失败, sleep 5 秒后重新获取
+  def flush(events, final)
     wio = StringIO.new("w")
     gzip_io = Zlib::GzipWriter.new(wio)
     gzip_io.write(events.to_json)
@@ -96,11 +274,56 @@ class LogStash::Outputs::SensorsAnalytics < LogStash::Outputs::Base
     data = Base64.strict_encode64(wio.string)
     form_data = {"data_list" => data, "gzip" => 1}
 
-    response = client.post(@url, :params => form_data).call
-    if response.code != 200
-      @logger.warn("Send failed, code: #{response.code}, body: #{response.body}")
-      raise
+    url_item = obtain_url
+
+    until do_send(form_data, url_item[:url])
+      last_url = url_item[:url]
+      # 将发送失败的 url 标记为不可用
+      disable_url(url_item)
+      url_item = obtain_url
+      @logger.warn("Send failed, retry send data to another url", :last_url => last_url, :retry_url => url_item[:url])
     end
+    @url_send_count[url_item[:url]] += events.length
   end
 
-end # class LogStash::Outputs::SensorsAnalytics
+  private
+
+  # 把当前 buffer 用的 url 从 list 的 0 索引开始依次放入, 方便在 obtain_url 遍历
+  def init_url_list(urls, start_index)
+    @url_list = []
+    index = start_index
+    loop do
+      @url_list << {
+          :url => urls[index],
+          :ok? => true,
+          :fail_time => Time.now
+      }
+      index = (index + 1) % urls.length
+      break if index == start_index
+    end
+  end
+
+
+  private
+
+  def obtain_url
+    while true do
+      @url_list.each do |url_item|
+        return url_item if url_item[:ok?]
+        if Time.now - url_item[:fail_time] > 3
+          url_item[:ok] = true
+          return url_item
+        end
+      end
+      @logger.warn("All url disable, sleep 5 sec")
+      sleep 5
+    end
+  end
+
+  private
+
+  def disable_url(url_item)
+    url_item[:ok?] = false
+    url_item[:fail_time] = Time.now
+  end
+end

data/logstash-output-sensors_analytics.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-sensors_analytics'
-  s.version       = '0.1.0'
+  s.version       = '0.1.2'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'Output plugin for Sensors Analytics'
   s.description   = 'This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program.'

data/spec/outputs/sensors_analytics_spec.rb

@@ -3,6 +3,7 @@ require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/sensors_analytics"
 require "logstash/codecs/plain"
 require "logstash/event"
+require "base64"
 
 PORT = rand(65535 - 1024) + 1025
 
@@ -19,6 +20,38 @@ class TestApp < Sinatra::Base
     end
   end
 
+  def self.receive_count=(count)
+    @receive_count = count
+  end
+
+  def self.receive_count
+    @receive_count || 0
+  end
+
+  def self.mult_0=(count)
+    @mult_0 = count
+  end
+
+  def self.mult_0
+    @mult_0 || 0
+  end
+
+  def self.mult_1=(count)
+    @mult_1 = count
+  end
+
+  def self.mult_1
+    @mult_1 || 0
+  end
+
+  def self.mult_2=(count)
+    @mult_2 = count
+  end
+
+  def self.mult_2
+    @mult_2 || 0
+  end
+
   def self.last_request=(request)
     @last_request = request
   end
@@ -53,6 +86,28 @@ class TestApp < Sinatra::Base
 
   multiroute(%w(post), "/good") do
     self.class.last_request = request
+    self.class.receive_count += 1
+    [200, "YUP"]
+  end
+
+  multiroute(%w(post), "/mult_0") do
+    self.class.last_request = request
+    self.class.receive_count += 1
+    self.class.mult_0 += 1
+    [200, "YUP"]
+  end
+
+  multiroute(%w(post), "/mult_1") do
+    self.class.last_request = request
+    self.class.receive_count += 1
+    self.class.mult_1 += 1
+    [200, "YUP"]
+  end
+
+  multiroute(%w(post), "/mult_2") do
+    self.class.last_request = request
+    self.class.receive_count += 1
+    self.class.mult_2 += 1
     [200, "YUP"]
   end
 
@@ -105,9 +160,20 @@ end
 
 describe LogStash::Outputs::SensorsAnalytics do
 
+  # 解析 request body
+  def parse_data(request_data)
+    data = request_data[request_data.index("=") + 1...request_data.index("&gzip")]
+    url_decode = URI::decode(data)
+    base_64 = Base64.strict_decode64(url_decode)
+    io = StringIO.new(base_64)
+    gzip_io = Zlib::GzipReader.new(io)
+    json_str = gzip_io.read
+    JSON.parse(json_str)
+  end
+
   describe "basic test" do
 
-    let(:port) {PORT}
+    let(:port) { PORT }
     let(:event) {
       event = LogStash::Event.new
       event.set("path", "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
@@ -115,12 +181,12 @@ describe LogStash::Outputs::SensorsAnalytics do
       event.set("message", '{"distinct_id":"123456","time":1434556935000,"type":"track","event":"ViewProduct","properties":{"product_id":12345,"product_name":"苹果","product_classify":"水果","product_price":14}}')
       event
     }
-    let(:method) {"post"}
+    let(:method) { "post" }
 
     context 'sending no events' do
-      let(:url) {"http://localhost:#{port}/good"}
-      let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1}}
-      subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
+      let(:url) { "http://localhost:#{port}/good" }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
 
       before do
         subject.register
@@ -131,39 +197,171 @@ describe LogStash::Outputs::SensorsAnalytics do
       end
     end
 
+    context "send events" do
+      let(:url) { "http://localhost:#{port}/good" }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 10, "flush_interval_sec" => 2} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
+
+      before do
+        subject.register
+        TestApp.receive_count = 0
+      end
+
+      it 'should send requests 3 times' do
+        30.times { subject.multi_receive([event]) }
+        expect(TestApp.receive_count).to eq(3)
+      end
+
+      before do
+        TestApp.receive_count = 0
+      end
+
+      it 'should send the request after 2 seconds' do
+        7.times { subject.multi_receive([event]) }
+        expect(TestApp.receive_count).to eq(0)
+        sleep(3)
+        expect(TestApp.receive_count).to eq(1)
+      end
+    end
+
     context "with retryable failing requests" do
-      let(:url) {"http://localhost:#{port}/retry"}
-      let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "flush_interval" => 100}}
-      subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
+      let(:url) { "http://localhost:#{port}/retry" }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "flush_interval_sec" => 100} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
 
       before do
         subject.register
       end
 
       before do
-        TestApp.retry_fail_count = 3
+        TestApp.retry_fail_count = 5
+        event.set("@metadata", {"beat" => "filebeat"})
+        event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
+        event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
         subject.multi_receive([event])
       end
 
       it "should log a retryable response 6 times" do
+        expect(TestApp.retry_action_count).to eq(6)
         expect(TestApp.final_success).to eq(true)
-        expect(TestApp.retry_action_count).to eq(4)
-        expect(TestApp.last_request.body.read).to eq('data_list=H4sIAAAAAAAA%2F1WPQU7DMBBF7zJlmcZJ2xSRJWuQ2MAGoWqwp%2B20bhzZ06Kqygm4AuIMLNhwoYpjYEe0iJX93x%2F9P%2FN4AMNBuNEyYwM1lKPxpJpCBsIbgrqcRFlNr8ZVURQR7tsIQTzqdZyhHTUS9QPTy513Zqsl0ta7lrwwBagPSSXep%2Ffh2Rk1mCrg%2B%2FXr%2BP4Gf1xbDIHn%2B%2BgdPz7%2Fe61n3e%2FVZWD5OTVc9C%2FcuEUQDMs4nMhsRz6wa6JT5GVenvCGZOnSpeKcDSdqSJBtSsF2K2joFvW1c%2BthPCu3TqMdDNR9iJFqTs1ixbhiUn2Esr%2FF58%2FwMhUqg4IjhdB1Tz8MgoViZwEAAA%3D%3D&gzip=1')
+        event_obj = parse_data(TestApp.last_request.body.read)[0]
+        expect(event_obj["event"]).to eq("ViewProduct")
+        expect(event_obj["distinct_id"]).to eq("123456")
+        expect(event_obj["lib"]["$lib_detail"])
+            .to eq("LaputadeMacBook-Pro.local##/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a")
       end
     end
 
     context 'config project' do
-      let(:url) {"http://localhost:#{port}/good"}
-      let(:verb_behavior_config) {{"url" => url, "flush_batch_size" => 1, "project" => "production"}}
-      subject {LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config)}
+      let(:url) { "http://localhost:#{port}/good" }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1, "project" => "production"} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
 
       before do
+        event.set("@metadata", {"beat" => "filebeat"})
+        event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
+        event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/a"}})
         subject.register
         subject.multi_receive([event])
       end
 
       it 'should add project into record' do
-        expect(TestApp.last_request.body.read).to eq('data_list=H4sIAAAAAAAA%2F1WQTU7DQAyF7%2BKyTDNJ2xSRJWuQ2MAGocpM3NbpNBNl3KKqygm4AuIMLNhwoYpjMDPqj1jN%2BLP13rOf91CxE260zLiCEvLReFJMIQHhNUGZT3xZTG%2FGRZZlHu5aD0E61Cs%2FQ1tqxNdPTG8Pna02WjxtO9tSJ0wOyn2oAo%2FqUTw5owaDBfy%2B%2Fxw%2BP%2BDCtUHneL7zvcPX9%2F9e27GOufoEDL8Gh6v4wp1dOEG39MOBzLbUObaN72RpnuYnvCZZ2rCpWGvciVYkyCaoYLsRrOge9a21q6FfKzVWoxkM1KPzkmpOzaJmrJlUlFDmaHz%2BDK%2BDoapQcKQQ%2Bpi%2BJh1OddwjBOtf%2FgAJu5tJfgEAAA%3D%3D&gzip=1')
+        event_obj = parse_data(TestApp.last_request.body.read)[0]
+        expect(event_obj["project"]).to eq("production")
+      end
+    end
+
+    context 'config multiple url' do
+      let(:url) { Array["http://localhost:#{port}/mult_0",
+                        "http://localhost:#{port}/mult_1",
+                        "http://localhost:#{port}/mult_2"] }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
+
+      # filebeat input 时默认通过 hostname + path 做 hash
+      before do
+        TestApp.mult_0 = 0
+        TestApp.mult_1 = 0
+        TestApp.mult_1 = 0
+        TestApp.receive_count = 0
+        subject.register
+        event.set("@metadata", {"beat" => "filebeat"})
+        event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
+        (0...100).each { |i|
+          event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
+          subject.multi_receive([event])
+        }
+      end
+
+      it 'should send event to multiple url' do
+        expect(TestApp.mult_0 > 10).to eq(true)
+        expect(TestApp.mult_1 > 10).to eq(true)
+        expect(TestApp.mult_2 > 10).to eq(true)
+        expect(TestApp.receive_count).to eq(100)
+      end
+    end
+
+    context 'config multiple url include bad url' do
+      let(:url) { Array["http://localhost:#{port}/mult_0",
+                        "http://localhost:#{port}/mult_1",
+                        "http://localhost:#{port}/mult_2",
+                        "http://localhost:#{port}/bad"] }
+      let(:verb_behavior_config) { {"url" => url, "flush_batch_size" => 1} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
+
+      # filebeat input 时默认通过 hostname + path 做 hash
+      before do
+        TestApp.mult_0 = 0
+        TestApp.mult_1 = 0
+        TestApp.mult_1 = 0
+        TestApp.receive_count = 0
+        subject.register
+        event.set("@metadata", {"beat" => "filebeat"})
+        event.set("host", {"name" => "LaputadeMacBook-Pro.local"})
+        (0...100).each { |i|
+          event.set("log", {"file" => {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"}})
+          subject.multi_receive([event])
+        }
+        subject.close
+      end
+
+      it 'should send event to multiple url exclude bad url' do
+        expect(TestApp.mult_0 > 15).to eq(true)
+        expect(TestApp.mult_1 > 15).to eq(true)
+        expect(TestApp.mult_2 > 15).to eq(true)
+        expect(TestApp.receive_count).to eq(100)
+      end
+    end
+
+    context 'config hash filed and send to multiple url include bad url' do
+      let(:url) { Array["http://localhost:#{port}/mult_0",
+                        "http://localhost:#{port}/mult_1",
+                        "http://localhost:#{port}/mult_2",
+                        "http://localhost:#{port}/bad"] }
+      let(:verb_behavior_config) { {"url" => url,
+                                    "flush_batch_size" => 1,
+                                    "hash_filed" => Array["name", "[@metadata][path]"]} }
+      subject { LogStash::Outputs::SensorsAnalytics.new(verb_behavior_config) }
+
+      # filebeat input 时默认通过 hostname + path 做 hash
+      before do
+        TestApp.mult_0 = 0
+        TestApp.mult_1 = 0
+        TestApp.mult_1 = 0
+        TestApp.receive_count = 0
+        subject.register
+        (0...100).each { |i|
+          event.set("name", "name#{i}")
+          event.set("@metadata", {"path" => "/Users/fengjiajie/tools/logstash/logstash-7.1.1/data2/#{i}"})
+          subject.multi_receive([event])
+        }
+      end
+
+      it 'should send event to multiple url exclude bad url' do
+        expect(TestApp.mult_0 > 10).to eq(true)
+        expect(TestApp.mult_1 > 10).to eq(true)
+        expect(TestApp.mult_2 > 10).to eq(true)
+        expect(TestApp.receive_count).to eq(100)
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-sensors_analytics
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Feng Jiajie
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-07-20 00:00:00.000000000 Z
+date: 2020-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement