logstash-output-logservice 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5ceec20c00719163e5c5973e3ec5adb32f656ac5
-  data.tar.gz: ca1fe9f695e8fb80f4ec5919ea0c4d4294b4f55a
+  metadata.gz: 8bef3eb05efe5d795d632577235795820512cc68
+  data.tar.gz: 28c0713b27c7cf0088a0125be5311625b08a44cb
 SHA512:
-  metadata.gz: 6dee2854d218ab0cd9b53477c967a40096488500862292c31ee41a22ad181678769b5656a71324c46908d9a52e96c37ba8db287b1c6c6d22a2d2a4c01c8ec738
-  data.tar.gz: 3f0caa777fcec257ee21d0f8b8100617678f2c7027a241ae95c9ceb253f8d99b217d8d0b0032c1abad6bc6e93af32d20403ef8c05ce95c8b8821897477555a35
+  metadata.gz: 5ba0d1d3d05beabe056af97f88fc98b7cb7b217611d0e63ad5d18b61f971c32a383b4b90526d0a9160004041753410b06196cc8d63ebcd954e1126c1d7f6801c
+  data.tar.gz: 1d1b435298e30f8879a732f1caaff9d8a605dcc3554efa936d45589cf687d690a257264d04ca62ebe25e9445f7d9c9765899c8b212a60d19c2f506fa0fcaa9b6

data/lib/logstash/outputs/logservice.rb

@@ -7,6 +7,7 @@ require 'socket'
 require "java"
 
 import "java.util.ArrayList"
+import "java.util.HashMap"
 
 root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
 
@@ -21,16 +22,16 @@ class LogStash::Outputs::LogService < LogStash::Outputs::Base
   config :endpoint, :validate => :string, :required => true
   config :project, :validate=> :string, :required=> true
   config :logstore, :validate=> :string, :required=> true
-  config :topic, :validate=> :string, :requried=> false, :default=> ""
+  config :topic, :validate=> :string, :required=> false, :default=> ""
   # if source is null, will set ip default
-  config :source, :validete=> :string, :required=> true
+  config :source, :validate=> :string, :required=> true
 
   # access_key_id/access_key_secret created by account of aliyun.com
   config :access_key_id, :validate=> :string, :required=> true
   config :access_key_secret, :validate=> :string, :required=> true
 
   # default 1000 logs in a logGroup for batch send
-  config :max_buffer_items, :validate=> :number, :required=> false, :default=> 1000
+  config :max_buffer_items, :validate=> :number, :required=> false, :default=> 4000
   # default 2*1024*1024 Bytes in a logGroup for batch send
   config :max_buffer_bytes, :validate=> :number, :required=> false, :default=> 2097152
   # for batch send, logGroup will emit in default 3 seconds
@@ -57,6 +58,16 @@ class LogStash::Outputs::LogService < LogStash::Outputs::Base
       @source = Socket.ip_address_list.detect{|intf| intf.ipv4_private?}
     end
     @send_retry_interval_seconds = @send_retry_interval / 1000.0
+    @topic_regex = nil
+    if ( @topic =~ /.*\(.*\).*/ )
+        begin
+            @topic_regex = Regexp.new(@topic)
+        rescue
+            @logger.error("topic is not valid regex", :topic => @topic)
+            @topic_regex = nil
+        end
+    end
+    
     @logger.info("init logstash-output-logservice plugin", :endpoint => @endpoint, :project => @project, :logstore => @logstore, :topic => @topic, :source => @source, :max_buffer_bytes => @max_buffer_bytes)
   end # def register
 
@@ -70,11 +81,11 @@ class LogStash::Outputs::LogService < LogStash::Outputs::Base
     end
   end # def event
 
-  def send_to_log_service(loggroup)
+  def send_to_log_service(loggroup, loggroup_topic)
     @retry = 0
     begin
       @retry += 1
-      @logclient.PutLogs(@project, @logstore, @topic, loggroup, @source)
+      @logclient.PutLogs(@project, @logstore, loggroup_topic, loggroup, @source)
       @logger.info("send logs to logservice success", :logcount => loggroup.size().to_s)
     rescue LogException => e
       @error_code = e.GetErrorCode()
@@ -103,36 +114,63 @@ class LogStash::Outputs::LogService < LogStash::Outputs::Base
       return
     end
 
-    @loggroup = ArrayList.new
-    @byte_size = 0
+    @topic_map = HashMap.new()
+    @topic_size_map = HashMap.new() 
     events.each { |x|
       begin
         @event_map = x.to_hash
         if @event_map.size < 1
           next
         end
-        @logitem = LogCommon.LogItem.new
+
+        @loggroup_topic = @topic 
+        if @topic_regex != nil and @event_map.has_key?('path')
+          @topic_match = @topic_regex.match(@event_map['path'].to_s)
+          if @topic_match != nil and @topic_match.length > 1
+            @loggroup_topic = @topic_match[1]
+          else
+            @loggroup_topic = ""
+          end
+        end
+
+        @logitem = LogCommon.LogItem.new()
         #@timestamp like 2016-02-18T03:23:11.053Z
         @logitem.SetTime(Time.parse(@event_map['@timestamp'].to_s).to_i)
+        @byte_size = 0
         @event_map.each do | key, value |
           @key_str = key.to_s
           @value_str = value.to_s
           @byte_size += @key_str.length + @value_str.length
           @logitem.PushBack(@key_str, @value_str)
         end
-        @loggroup.add(@logitem)
-        if @byte_size > @max_buffer_bytes
-          send_to_log_service(@loggroup)
-          @loggroup = ArrayList.new
-          @byte_size = 0
+
+        if (@topic_map.containsKey(@loggroup_topic))
+          @topic_map[@loggroup_topic].add(@logitem)
+          @topic_size_map[@loggroup_topic] += @byte_size
+        else
+          @loggroup = ArrayList.new()
+          @loggroup.add(@logitem)
+          @topic_map[@loggroup_topic] = @loggroup
+          @topic_size_map[@loggroup_topic] = @byte_size
+        end
+
+        if @topic_size_map[@loggroup_topic] >= @max_buffer_bytes
+          @logger.debug("flush loggroup", :loggroup_topic => @loggroup_topic, :bytes => @topic_size_map[@loggroup_topic], :logs => @topic_map[@loggroup_topic].size())
+          send_to_log_service(@topic_map[@loggroup_topic], @loggroup_topic)
+          @topic_map.remove(@loggroup_topic)
+          @topic_size_map.remove(@loggroup_topic)
         end
       rescue => e
         @logger.warn("decode log data to LogGroup fail", :exception => e)
       end
     }
-    if @byte_size > 0
-      send_to_log_service(@loggroup)
-    end
+
+    @topic_map.keySet().each { | key |
+      if @topic_size_map[key] > 0
+        @logger.debug("flush loggroup", :key => key, :bytes => @topic_size_map[key], :logs => @topic_map[key].size())
+        send_to_log_service(@topic_map[key], key)
+      end
+    }
   end
 
   public

data/logstash-output-logservice.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-logservice'
-  s.version = "0.2.0"
+  s.version = "0.3.0"
   s.licenses = ['Apache License (2.0)']
   s.summary = "put data into logservice."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "stud"
   s.add_development_dependency "logstash-devutils"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-logservice
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - tangkai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-29 00:00:00.000000000 Z
+date: 2017-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -17,9 +17,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,9 +24,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
@@ -129,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: put data into logservice.