logstash-output-logservice 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5448b1cff9f51df78207689cdbe6e9f8f9588c58d839fcbde73cfafffa291010
-  data.tar.gz: 4c38fd10f368e89a8c258923b8054c4e16a978ac63fa88732640b921f9498b34
+  metadata.gz: b0bb3e1fa2c186f90e1abb53d263621a7f9a59c8af69acb288133bd5c9f92d4a
+  data.tar.gz: 8d1f51eb419826ea7f3e32979d5635fedbf3cd02ab9815d07e536e086d4972e3
 SHA512:
-  metadata.gz: deed2ea214b113fa452ddb89b9ae8a1124f1bd5e1711c4d79e95e76f3d3780655fc4b7f6c3550d8792f99cf9b7358782fe1eaa20b6a6c6b9d78db2f49d7331e3
-  data.tar.gz: 530e5cdc3ccb5f80b438db7994fe51a00ac3decfc365dd27ce6765d662cdce9370c116b264213be5df05aa1f9f86b960bb97546e23c8a653df646f9a72c615ae
+  metadata.gz: 30fc3c1789a4995761958cd82f402533e92e51716b71a8a8da06ccfba720eb6a15055de295f97f44d988755d2b585b599a2a199fe1cee865da1c6de7eb31f14a
+  data.tar.gz: c35a79edaf9733f5164739be49bf682bb80bab26b5736e276b5a9cc781712198d890c242f32301ed56c1005c68a88dec94f96e60f525e033a05f644cf246e1e2

data/lib/logstash/outputs/logservice.rb

@@ -7,10 +7,7 @@ require 'socket'
 require "java"
 require 'json'
 
-import "java.util.ArrayList"
-
 root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
-
 LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")
 
 class LogStash::Outputs::LogService < LogStash::Outputs::Base
@@ -31,134 +28,80 @@ class LogStash::Outputs::LogService < LogStash::Outputs::Base
   config :access_key_secret, :validate=> :string, :required=> true
 
   # default 4000 logs in a logGroup for batch send
-  config :max_buffer_items, :validate=> :number, :required=> false, :default=> 4000
+  config :max_buffer_items, :validate=> :number, :required=> false, :default=> 4096
   # default 2*1024*1024 Bytes in a logGroup for batch send
   config :max_buffer_bytes, :validate=> :number, :required=> false, :default=> 2097152
   # for batch send, logGroup will emit in default 3 seconds
-  config :max_buffer_seconds, :validate=> :number, :required=> false, :default=> 3
+  config :max_buffer_seconds, :validate=> :number, :required=> false, :default=> 2
+  # the maximum log size that a single producer instance can cache is 100MB by default.
+  config :total_size_in_bytes, :validate=> :number, :required=> false, :default=> 104857600
 
   # logGroup will retry send to log service when error happened, and will be discard when retry times exceed limit
-  config :max_send_retry, :validate=> :number, :required=> true, :default=> 10 
+  config :max_send_retry, :validate=> :number, :required=> true, :default=> 10
   # sleep default 200 milliseconds before retry next send
   config :send_retry_interval, :validate=> :number, :required=> false, :default=> 200
 
   config :to_json, :validate=> :boolean, :required=> false, :default=> true
   config :time_key, :validate=> :string, :required=> false, :default=> "@timestamp"
 
-  Log = com.aliyun.openservices.log
-  LogException = com.aliyun.openservices.log.exception.LogException
-  LogCommon = com.aliyun.openservices.log.common
+  LP = com.aliyun.openservices.aliyun.log.producer
+  LogCommon = com.shade.aliyun.openservices.log.common
+
 
   public
   def register
-    buffer_initialize(
-        :max_items => @max_buffer_items,
-        :max_interval => @max_buffer_seconds,
-        :logger => @logger
-    )
-    @logclient = Log.Client::new(@endpoint, @access_key_id, @access_key_secret)
-#     if @source == ''
-#         @source = Socket.ip_address_list.detect{|intf| intf.ipv4_private?}.ip_address
-#     end
-    @send_retry_interval_seconds = @send_retry_interval / 1000.0
+    @producerConfig = LP.ProducerConfig::new();
+    @producerConfig.setBatchCountThreshold(@max_buffer_items);
+    @producerConfig.setBatchSizeThresholdInBytes(@max_buffer_bytes);
+    @producerConfig.setLingerMs(@max_buffer_seconds*1000);
+    @producerConfig.setRetries(@max_send_retry);
+    @producerConfig.setBaseRetryBackoffMs(@send_retry_interval);
+    @producerConfig.setTotalSizeInBytes(@total_size_in_bytes);
+    @producer = LP.LogProducer::new(@producerConfig);
+    @producer.putProjectConfig(LP.ProjectConfig::new(@project, @endpoint, @access_key_id,  @access_key_secret));
+
     @logger.info("init logstash-output-logservice plugin", :endpoint => @endpoint, :project => @project, :logstore => @logstore, :topic => @topic, :source => @source, :max_buffer_bytes => @max_buffer_bytes)
   end # def register
 
   public
   def receive(event)
-    return unless output?(event)
     begin
-      buffer_receive(event)
-    rescue => e
-      @logger.warn("error encoding event for logstash-output-logservice", :exception => e, :event => event)
-    end
-  end # def event
-
-  def send_to_log_service(loggroup)
-    @retry = 0
-    begin
-      @retry += 1
-      @logclient.PutLogs(@project, @logstore, @topic, loggroup, @source.to_s)
-      @logger.info("send logs to logservice success", :logcount => loggroup.size().to_s)
-    rescue LogException => e
-      @error_code = e.GetErrorCode()
-      @error_message = e.GetErrorMessage()
-      if @retry < @max_send_retry
-        @logger.warn("send logs to logservice fail, will retry later", :exception => e, :error_code => @error_code, :error_message => @error_message, :retry => @retry)
-        sleep(@send_retry_interval_seconds)
-        retry
-      else
-        @logger.error("send logs to logservice fail, discard data", :exception => e, :error_code => @error_code, :error_message => @error_message, :retry => @retry)
+      @event_map = event.to_hash
+      if @event_map.size < 1
+        return
       end
-    rescue => e
-      if @retry < @max_send_retry
-        @logger.warn("send logs to logservice fail, retry later", :exception => e, :retry => @retry)
-        sleep(@send_retry_interval_seconds)
-        retry
-      else
-        @logger.error("send logs to logservice fail, discard data", :exception => e, :retry => @retry)
+      @logitem = LogCommon.LogItem.new
+      #@timestamp like 2016-02-18T03:23:11.053Z
+      time_value = @event_map[@time_key]
+      if time_value.nil?
+         time_value = @event_map['@timestamp']
+         @logger.warn("The time_key is nil, use @timestamp")
       end
-    end
-  end
-
-  # called from Stud::Buffer#buffer_flush when there are events to flush
-  def flush(events, close=false)
-    if events.size < 1
-      return
-    end
-
-    @loggroup = ArrayList.new
-    @byte_size = 0
-    events.each { |x|
-      begin
-        @event_map = x.to_hash
-        if @event_map.size < 1
+      @logitem.SetTime(Time.parse(time_value.to_s).to_i)
+      @event_map.each do | key, value |
+        @key_str = key.to_s
+        if @key_str == '__time__'
           next
         end
-        @logitem = LogCommon.LogItem.new
-        #@timestamp like 2016-02-18T03:23:11.053Z
-        time_value = @event_map[@time_key]
-        if time_value.nil?
-           time_value = @event_map['@timestamp']
-           @logger.warn("The time_key is nil, use @timestamp")
-        end
-        @logitem.SetTime(Time.parse(time_value.to_s).to_i)
-        @event_map.each do | key, value |
-          @key_str = key.to_s
-          if @key_str == '__time__'
-            next
-          end
-          if value.instance_of? Hash
-            @value_str = value.to_json
-          else
-            @value_str = value.to_s
-          end
-#           if @to_json
-#             @value_str = value.to_json
-#           else
-#             @value_str = value.to_s
-#           end
-          @byte_size += @key_str.length + @value_str.length
-          @logitem.PushBack(@key_str, @value_str)
-        end
-        @loggroup.add(@logitem)
-        if @byte_size > @max_buffer_bytes
-          send_to_log_service(@loggroup)
-          @loggroup = ArrayList.new
-          @byte_size = 0
+        if value.instance_of? Hash
+          @value_str = value.to_json
+        else
+          @value_str = value.to_s
         end
+        @logitem.PushBack(@key_str, @value_str)
+      end
+      @producer.send(@project, @logstore, @topic, @source, @logitem)
       rescue => e
-        @logger.warn("deserialize log data from json to LogGroup(protobuf) fail", :exception => e)
+        @logger.warn("send log data fail", :exception => e)
       end
-    }
-    if @byte_size > 0
-      send_to_log_service(@loggroup)
-    end
+    end # def event
+
+  def flush(events, close=false)
   end
 
   public
   def close
-    buffer_flush(:final => true)
+    @producer.close();
   end # def close
 
-end # class LogStash::Outputs::LogService
+end # class LogStash::Outputs::LogService

data/logstash-output-logservice.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-logservice'
-  s.version = "0.14.0"
+  s.version = "0.15.0"
   s.licenses = ['Apache-2.0']
   s.summary = "post data into logservice."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-logservice
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - tangkai
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-11-22 00:00:00.000000000 Z
+date: 2023-11-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -84,14 +84,7 @@ files:
 - LICENSE
 - README.md
 - lib/logstash/outputs/logservice.rb
-- lib/logstash/vendor/jar-dependencies/runtime-jars/aliyun-log-private-cloud-0.6.73.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/commons-logging-1.2.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/gson-2.9.0.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/httpclient-4.5.1.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/httpcore-4.4.3.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
-- lib/logstash/vendor/jar-dependencies/runtime-jars/protobuf-java-2.5.0.jar
+- lib/logstash/vendor/jar-dependencies/runtime-jars/aliyun-log-producer-0.3.17-jar-with-dependencies.jar
 - logstash-output-logservice.gemspec
 - spec/outputs/logservice_spec.rb
 homepage: https://www.aliyun.com/product/sls
@@ -115,7 +108,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: post data into logservice.