logstash-integration-aws 7.3.2-java → 7.3.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e1b8a5b5a5fc9020b5c91a867bf8608cc465ca404bcab2e8fbde85e36768aa63
-  data.tar.gz: 0a5a55593fab48534744b17df0c6ba14b2b63b5d63ea2437a85b40d7421bf9d8
+  metadata.gz: 66d4cda26d968b67291bf9864364e51ca118b67646c4145e75d612ee30516b24
+  data.tar.gz: f2145daff4c36a702eeed4b4e22e6b4304dfef8ac0d5e3fed50a2dc9d86e94e2
 SHA512:
-  metadata.gz: 43bfee142660e88a973349580daae1d933ff17721225e2fe6fd3592ec02306498f359d4497d34f677c17be903c4e521bdd2095ed0ab36da9c7ff5ebce74bba4f
-  data.tar.gz: d692f03255dd68b9e0d0b6f6d42ef7db17d11e49d116d00ebc761702ad5f36a65e6e1648f8f25c27f58b82576bc57af9b54f2aa8ff7ef9b5359028d260cf8dd2
+  metadata.gz: 99debf93aa83e1bd10a638e80e9693f0500139f35f073afe22a2b10d486db88dc2a81a3ad4a566606a4fd0ab333bcdbb99d0efbc2c6f72e7c9165cbbb673ae9a
+  data.tar.gz: 3bfe2df657134abe85d4469d77569ba6c823017602576c9e0ddb8d24815b7f44bee89e6277cbad8d072bf9aad4480d8dd1f424a2a1cb4a729a4cfe3dd0d10240

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 7.3.4
+  - Use milliseconds timestamp precision in S3 input to fix the skip backup and delete object issue in S3-compatible storage services [#60](https://github.com/logstash-plugins/logstash-integration-aws/pull/60)
+    
+## 7.3.3
+  - Replace deprecated `Aws::S3::Object#upload_file` in favor of `Aws::S3::TransferManager#upload_file` [#67](https://github.com/logstash-plugins/logstash-integration-aws/pull/67)
+
 ## 7.3.2
   - Fix: replace deprecated `File.exists?` with `File.exist?` for Ruby 3.4 (JRuby 10) compatibility [#65](https://github.com/logstash-plugins/logstash-integration-aws/pull/65)
 

data/VERSION

@@ -1 +1 @@
-7.3.2
+7.3.4

data/lib/logstash/inputs/s3.rb

@@ -148,7 +148,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
           @logger.debug('Ignoring', :key => log.key)
         elsif log.content_length <= 0
           @logger.debug('Object Zero Length', :key => log.key)
-        elsif log.last_modified <= sincedb_time
+        elsif log.last_modified.to_i <= sincedb_time.to_i
           @logger.debug('Object Not Modified', :key => log.key)
         elsif log.last_modified > (current_time - @cutoff_second).utc # recently modified files will be processed in next cycle
           @logger.debug('Object Modified After Cutoff Time', :key => log.key)
@@ -380,7 +380,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     filename = File.join(temporary_directory, File.basename(log.key))
     if download_remote_file(object, filename)
       if process_local_log(queue, filename, object)
-        if object.last_modified == log.last_modified
+        if object.last_modified.to_i == log.last_modified.to_i
           backup_to_bucket(object)
           backup_to_dir(filename)
           delete_file_from_bucket(object)

data/lib/logstash/outputs/s3/uploader.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 require "logstash/util"
-require "aws-sdk-core"
+require "aws-sdk-s3"
 
 module LogStash
   module Outputs
@@ -18,6 +18,7 @@ module LogStash
 
         def initialize(bucket, logger, threadpool = DEFAULT_THREADPOOL, retry_count: Float::INFINITY, retry_delay: 1)
           @bucket = bucket
+          @transfer_manager = Aws::S3::TransferManager.new(client: bucket.client)
           @workers_pool = threadpool
           @logger = logger
           @retry_count = retry_count
@@ -37,8 +38,7 @@ module LogStash
 
           tries = 0
           begin
-            obj = bucket.object(file.key)
-            obj.upload_file(file.path, upload_options)
+            @transfer_manager.upload_file(file.path, bucket: bucket.name, key: file.key, **upload_options)
           rescue Errno::ENOENT => e
             logger.error("File doesn't exist! Unrecoverable error.", :exception => e.class, :message => e.message, :path => file.path, :backtrace => e.backtrace)
           rescue => e

data/lib/logstash/outputs/s3/write_bucket_permission_validator.rb

@@ -2,6 +2,7 @@
 require "stud/temporary"
 require "socket"
 require "fileutils"
+require "aws-sdk-s3"
 
 module LogStash
   module Outputs
@@ -39,11 +40,11 @@ module LogStash
             f.write(content)
             f.fsync
 
-            obj = bucket_resource.object(key)
-            obj.upload_file(f, upload_options)
+            transfer_manager = Aws::S3::TransferManager.new(client: bucket_resource.client)
+            transfer_manager.upload_file(f.path, bucket: bucket_resource.name, key: key, **upload_options)
 
             begin
-              obj.delete
+              bucket_resource.object(key).delete
             rescue
               # Try to remove the files on the remote bucket,
               # but don't raise any errors if that doesn't work.

data/lib/logstash-integration-aws_jars.rb

@@ -1,4 +1,4 @@
 # AUTOGENERATED BY THE GRADLE SCRIPT. DO NOT EDIT.
 
 require 'jar_dependencies'
-require_jar('org.logstash.plugins.integration.aws', 'logstash-integration-aws', '7.3.2')
+require_jar('org.logstash.plugins.integration.aws', 'logstash-integration-aws', '7.3.4')

data/spec/inputs/s3_spec.rb

@@ -412,28 +412,14 @@ describe LogStash::Inputs::S3 do
 
     context "when event doesn't have a `message` field" do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json.log') }
-      let(:config) {
-        {
-          "access_key_id" => "1234",
-          "secret_access_key" => "secret",
-          "bucket" => "logstash-test",
-          "codec" => "json",
-        }
-      }
+      let(:config) { super().merge({ "codec" => "json" }) }
 
       include_examples "generated events"
     end
 
     context "when event does have a `message` field" do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json_with_message.log') }
-      let(:config) {
-        {
-          "access_key_id" => "1234",
-          "secret_access_key" => "secret",
-          "bucket" => "logstash-test",
-          "codec" => "json",
-        }
-      }
+      let(:config) { super().merge({ "codec" => "json" }) }
 
       include_examples "generated events"
     end
@@ -476,14 +462,11 @@ describe LogStash::Inputs::S3 do
 
     context 'multi-line' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiline.log') }
-       let(:config) {
-           {
-              "access_key_id" => "1234",
-              "secret_access_key" => "secret",
-              "bucket" => "logstash-test",
-              "codec" => LogStash::Codecs::Multiline.new( {"pattern" => "__SEPARATOR__", "negate" => "true",  "what" => "previous"})
-           }
-        }
+      let(:config) {
+        super().merge({
+          "codec" => LogStash::Codecs::Multiline.new( {"pattern" => "__SEPARATOR__", "negate" => "true",  "what" => "previous"})
+        })
+      }
 
       include_examples "generated events"
     end

data/spec/outputs/s3/uploader_spec.rb

@@ -43,26 +43,22 @@ describe LogStash::Outputs::S3::Uploader do
   end
 
   it "retries errors indefinitely" do
-    s3 = double("s3").as_null_object
-
-    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+    tm = subject.instance_variable_get(:@transfer_manager)
 
     expect(logger).to receive(:warn).with(any_args)
-    expect(s3).to receive(:upload_file).with(any_args).and_raise(RuntimeError.new('UPLOAD FAILED')).exactly(5).times
-    expect(s3).to receive(:upload_file).with(any_args).and_return(true)
+    expect(tm).to receive(:upload_file).with(file.path, hash_including(bucket: bucket_name, key: file.key)).and_raise(RuntimeError.new('UPLOAD FAILED')).exactly(5).times
+    expect(tm).to receive(:upload_file).with(file.path, hash_including(bucket: bucket_name, key: file.key)).and_return(true)
 
     subject.upload(file)
   end
 
   it "retries errors specified times" do
     subject = described_class.new(bucket, logger, threadpool, retry_count: 3, retry_delay: 0.01)
-    s3 = double("s3").as_null_object
-
-    allow(bucket).to receive(:object).with(file.key).and_return(s3)
+    tm = subject.instance_variable_get(:@transfer_manager)
 
     expect(logger).to receive(:warn).with(any_args).exactly(3).times
     expect(logger).to receive(:error).with(any_args).once
-    expect(s3).to receive(:upload_file).with(file.path, {}).and_raise(RuntimeError).at_least(1).times
+    expect(tm).to receive(:upload_file).with(file.path, hash_including(bucket: bucket_name, key: file.key)).and_raise(RuntimeError).at_least(1).times
 
     subject.upload(file)
   end

data/spec/outputs/s3/write_bucket_permission_validator_spec.rb

@@ -1,11 +1,13 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
+require "aws-sdk-s3"
 require "logstash/outputs/s3/write_bucket_permission_validator"
 
 describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
   let(:logger) { spy(:logger ) }
   let(:bucket_name) { "foobar" }
   let(:obj) { double("s3_object") }
+  let(:tm) { double("transfer_manager") }
   let(:client) { Aws::S3::Client.new(stub_responses: true) }
   let(:bucket) { Aws::S3::Bucket.new(bucket_name, :client => client) }
   let(:upload_options) { {} }
@@ -13,13 +15,14 @@ describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
   subject { described_class.new(logger) }
 
   before do
-    expect(bucket).to receive(:object).with(any_args).and_return(obj)
+    allow(Aws::S3::TransferManager).to receive(:new).with(client: client).and_return(tm)
+    allow(bucket).to receive(:object).with(any_args).and_return(obj)
   end
 
   context 'when using upload_options' do
     let(:upload_options) {{ :server_side_encryption => true }}
     it 'they are passed through to upload_file' do
-      expect(obj).to receive(:upload_file).with(anything, upload_options)
+      expect(tm).to receive(:upload_file).with(anything, hash_including(upload_options))
       expect(obj).to receive(:delete).and_return(true)
       expect(subject.valid?(bucket, upload_options)).to be_truthy
     end
@@ -28,13 +31,13 @@ describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
 
   context "when permissions are sufficient" do
     it "returns true" do
-      expect(obj).to receive(:upload_file).with(any_args).and_return(true)
+      expect(tm).to receive(:upload_file).with(any_args).and_return(true)
       expect(obj).to receive(:delete).and_return(true)
       expect(subject.valid?(bucket, upload_options)).to be_truthy
     end
 
     it "hides delete errors" do
-      expect(obj).to receive(:upload_file).with(any_args).and_return(true)
+      expect(tm).to receive(:upload_file).with(any_args).and_return(true)
       expect(obj).to receive(:delete).and_raise(StandardError)
       expect(subject.valid?(bucket, upload_options)).to be_truthy
     end
@@ -42,7 +45,7 @@ describe LogStash::Outputs::S3::WriteBucketPermissionValidator do
 
   context "when permission aren't sufficient" do
     it "returns false" do
-      expect(obj).to receive(:upload_file).with(any_args).and_raise(StandardError)
+      expect(tm).to receive(:upload_file).with(any_args).and_raise(StandardError)
       expect(subject.valid?(bucket, upload_options)).to be_falsey
     end
   end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-aws
 version: !ruby/object:Gem::Version
-  version: 7.3.2
+  version: 7.3.4
 platform: java
 authors:
 - Elastic
 bindir: bin
 cert_chain: []
-date: 2026-02-11 00:00:00.000000000 Z
+date: 2026-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core-plugin-api
@@ -407,7 +407,7 @@ files:
 - spec/spec_helper.rb
 - spec/support/helpers.rb
 - spec/unit/outputs/sqs_spec.rb
-- vendor/jar-dependencies/org/logstash/plugins/integration/aws/logstash-integration-aws/7.3.2/logstash-integration-aws-7.3.2.jar
+- vendor/jar-dependencies/org/logstash/plugins/integration/aws/logstash-integration-aws/7.3.4/logstash-integration-aws-7.3.4.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache-2.0