logstash-output-awslogs 1.1.2 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3f29bdc8b680f033bac338ae4613bc72e73403fbc7664c4c8eba78a0f497fdf3
-  data.tar.gz: 9b1e38924b1a1994ef71b69a4475f01d99e72345182b5aa1e8a0194b84cf98d6
+  metadata.gz: 5d89bf2683352b8d8e1dfb360760e5129db4d0bcfb447bea11101c3dc7648b11
+  data.tar.gz: 64e3749306d9da99ff81ca006bfb2e7c34255981b75119ee238d00b149e379f9
 SHA512:
-  metadata.gz: 9900328521269d53376e645216989c7c01647f26c1290b8dc88edd5732521a0555fadeb00ad0f4b9674c8b5336b6b550bf62715e92e3be5cd4ec3b5142292cca
-  data.tar.gz: 51d83fb4c44a8850b848dbc4900f96db80c0fa2521b7b9cf4726bfed3cf97ef7cb72d45ea69413d8f54062fc81773789d58c07177349690fa3766f9b92a7c548
+  metadata.gz: fc23ff3640ec9526027a3a1de59329dc00fedf2722602eca9e71ac4081007ad029e38391fe40a9895821aecd3e0f938c7f4f92c9f3139bee3a6410a640e8fffc
+  data.tar.gz: ce9e61a5e8d17bdb8aaa3a61bce0d7510f7547bc01327579263a9494cf110a8646d271d11b087bf915f035aa0de107f1a35eaa798f36306a546b9648ced32427

data/CHANGELOG.md

@@ -3,3 +3,7 @@
 
 ## 1.1.1
   - First fully working version.
+
+## 1.1.3
+  - Fixed working with big batches
+  - Sequence token removed

data/lib/logstash/outputs/awslogs.rb

@@ -4,7 +4,7 @@ require 'logstash/outputs/base'
 require 'logstash/namespace'
 require 'logstash/plugin_mixins/aws_config'
 require 'aws-sdk-cloudwatchlogs'
-require 'json'
+require 'time'
 
 
 class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
@@ -12,75 +12,116 @@ class LogStash::Outputs::Awslogs < LogStash::Outputs::Base
 
   config_name 'awslogs'
   default :codec, 'line'
+  concurrency :shared
 
+  PER_EVENT_OVERHEAD = 26
+  MAX_BATCH_SIZE = 1024 * 1024
+  MIN_DELAY = 0.2
+
+  # Log group to send event to
   config :log_group_name, validate: :string, required: true
+  # Logs stream to send event to
   config :log_stream_name, validate: :string, required: true
+  # Message to be sent. Fields interpolation supported. By default the whole event will be sent as a json object
+  config :message, validate: :string, default: ""
 
   public
   def register
     @client = Aws::CloudWatchLogs::Client.new(aws_options_hash)
-    @next_sequence_tokens = {}
+    @last_flush = Time.now.to_f
   end # def register
 
   public
   def multi_receive(events)
-    to_send = {}
+    send_batches = form_event_batches(events)
+    send_batches.each do |batch|
+      put_events(batch)
+    end
+  end
+
+  private
+  def put_events(batch)
+    begin
+      delay = MIN_DELAY - (Time.now.to_f - @last_flush)
+      sleep(delay) if delay > 0
+      @client.put_log_events(
+        {
+          log_group_name: batch[:log_group],
+          log_stream_name: batch[:log_stream],
+          log_events: batch[:log_events]
+        }
+      )
+    rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
+      @logger.info('AWSLogs: Will create log group/stream and retry')
+      begin
+        @client.create_log_group({log_group_name: batch[:log_group]})
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        @logger.info("AWSLogs: Log group #{batch[:log_group]} already exists")
+      end
+      begin
+        @client.create_log_stream({log_group_name: batch[:log_group], log_stream_name: batch[:log_stream]})
+      rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
+        @logger.info("AWSLogs: Log stream #{batch[:log_stream]} already exists")
+      end
+      retry
+    end
+    rescue Aws::CloudWatchLogs::Errors::LimitExceededException
+      @logger.info("AWSLogs: Rate limit exceeded, retrying")
+      retry
+    end
+    @last_flush = Time.now.to_f
+  end
 
-    events.each do |event|
+  private
+  def form_event_batches(events_arr)
+    batches = []
+    events_by_stream_and_group = {}
+    log_events = events_arr.sort_by { |event| event.timestamp.time.to_f}
+    log_events.each do |event|
       event_log_stream_name = event.sprintf(log_stream_name)
       event_log_group_name = event.sprintf(log_group_name)
 
-      next_sequence_token_key = [event_log_group_name, event_log_stream_name]
-      unless to_send.keys.include? next_sequence_token_key
-        to_send.store(next_sequence_token_key, [])
+      sort_key = [event_log_group_name, event_log_stream_name]
+      unless events_by_stream_and_group.keys.include? sort_key
+        events_by_stream_and_group.store(sort_key, [])
+      end
+      if message.empty?
+        events_by_stream_and_group[sort_key].push(
+          timestamp: (event.timestamp.time.to_f * 1000).to_int,
+          message: event.to_hash.sort.to_h.to_json
+        )
+      else
+        events_by_stream_and_group[sort_key].push(
+          timestamp: (event.timestamp.time.to_f * 1000).to_int,
+          message: event.sprintf(message)
+        )
       end
-      to_send[next_sequence_token_key].push(
-        timestamp: (event.timestamp.time.to_f * 1000).to_int,
-        message: event.to_hash.sort.to_h.to_json
-      )
     end
-
-
-    to_send.each do |event_log_names, log_events|
-      event_log_group_name = event_log_names[0]
-      event_log_stream_name = event_log_names[1]
-      next_sequence_token_key = [event_log_group_name, event_log_stream_name]
-
-      log_events.sort_by!{ |event| event[:timestamp] }
-
-      ident_opts = {
-        log_group_name: event_log_group_name,
-        log_stream_name: event_log_stream_name
+    events_by_stream_and_group.each do |key, value|
+      temp_batch = {
+        log_group: key[0],
+        log_stream: key[1],
+        size: 0,
+        log_events: []
       }
-      send_opts = ident_opts.merge(
-        log_events: log_events
-      )
-      if @next_sequence_tokens.keys.include? next_sequence_token_key
-        send_opts[:sequence_token] = @next_sequence_tokens[next_sequence_token_key]
-      end
-      begin
-        resp = @client.put_log_events(send_opts)
-        @next_sequence_tokens.store(next_sequence_token_key, resp.next_sequence_token)
-      rescue Aws::CloudWatchLogs::Errors::ResourceNotFoundException
-        @logger.info('Will create log group/stream and retry')
-        begin
-          @client.create_log_group({log_group_name: send_opts[:log_group_name]})
-        rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
-          @logger.info("Log group #{send_opts[:log_group_name]} already exists")
-        end
-        begin
-          @client.create_log_stream({log_group_name: send_opts[:log_group_name], log_stream_name: send_opts[:log_stream_name]})
-        rescue Aws::CloudWatchLogs::Errors::ResourceAlreadyExistsException
-          @logger.info("Log stream #{send_opts[:log_stream_name]} already exists")
+      value.each do |log_event|
+        if log_event[:message].bytesize + PER_EVENT_OVERHEAD + temp_batch[:size] < MAX_BATCH_SIZE
+          temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
+          temp_batch[:log_events] << log_event
+        else
+          batches << temp_batch
+          temp_batch = {
+            log_group: key[0],
+            log_stream: key[1],
+            size: 0,
+            log_events: []
+          }
+          temp_batch[:size] += log_event[:message].bytesize + PER_EVENT_OVERHEAD
+          temp_batch[:log_events] << log_event
         end
-        retry
-      rescue  Aws::CloudWatchLogs::Errors::InvalidSequenceTokenException => e
-        send_opts[:sequence_token] = e.expected_sequence_token
-        retry
-      rescue Aws::CloudWatchLogs::Errors::ThrottlingException
-        @logger.info('Logs throttling, retry')
-        retry
       end
+      batches << temp_batch
     end
-  end # def multi_receive_encodeds
-end # class LogStash::Outputs::Awslogs
+    batches
+  end
+end

data/logstash-output-awslogs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-awslogs'
-  s.version       = '1.1.2'
+  s.version       = '1.1.4'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Writes events to AWS CloudWatch logs.'
   s.homepage      = 'https://github.com/Anarhyst266/logstash-output-awslogs'
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_runtime_dependency "aws-sdk-cloudwatchlogs", '~> 1'
   s.add_runtime_dependency "logstash-integration-aws"

metadata

@@ -1,100 +1,106 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-awslogs
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.1.4
 platform: ruby
 authors:
 - Anton Klyba
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-12-12 00:00:00.000000000 Z
+date: 2024-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :runtime
+        version: 2.1.12
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.12
+    - - "<="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-codec-plain
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-cloudwatchlogs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
-  type: :runtime
+  name: aws-sdk-cloudwatchlogs
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: logstash-integration-aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: logstash-integration-aws
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: json
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: logstash-devutils
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: 
+description:
 email: anarhyst266+gems@gmail.com
 executables: []
 extensions: []
@@ -114,7 +120,7 @@ licenses:
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -129,8 +135,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3.1
-signing_key: 
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: Writes events to AWS CloudWatch logs.
 test_files: []