logstash-input-s3-sns-sqs 1.4.6 → 1.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b306531064b4c30538ebf2825ad8ccd3cffc1723
-  data.tar.gz: 3166705bd066e00607cfb5f8f6e0b5d810934ef2
+  metadata.gz: c857f29b851808997fb7403a5a1d4e71cd786e89
+  data.tar.gz: f2b27edba4192969a5c6e301ac6a19c0fabbeadf
 SHA512:
-  metadata.gz: ee7ba71f4ed496dc0cc109ea1834d0da58d112c320fb6c8a80f33ee65ab9a1f0746976be6ad32f934c4f80f9a3c0688f94a64064f44d665bbce3f3d19bb699b0
-  data.tar.gz: b614c2c3945819b6e95052f1770920d57b83d5afb9300836e6af1375b45c93c15c7117b14fdd33372edbe746cc796e5b967dc75b131b03d9eb0e7ba73cd65797
+  metadata.gz: a88ef9c5d505322a0639f5b77b74364ff64a9ea503ee969fad2389b38e82fa4064d9a5de85edd8767d80b0e8e9422a576e620831d63bee1c049da6e61137ab3d
+  data.tar.gz: 5a3fbe6296dbaee8a3c95d93ef6ba39f849326f1a138a95b949f351e4f570201b4af9a6db59899e90ef993b900a4a8f5cbc7d9587eb0420ee4df1561b46b766a

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 1.4.8
+- Bufix: CF Metadata events Bug #7
+- Feature: use aws-role for s3 client connection.
+## 1.4.7
+Remove from rubygems.org
 ## 1.4.6
 - BugFix: jRuby > 2 : No return from block
 - BugFix: No exit on gzip error

data/lib/logstash/inputs/s3snssqs.rb

@@ -114,6 +114,12 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
   # To run in multiple threads use this
   config :consumer_threads, :validate => :number, :default => 1
   config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash")
+  # The AWS IAM Role to assume, if any.
+  # This is used to generate temporary credentials typically for cross-account access.
+  # See https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html for more information.
+  config :s3_role_arn, :validate => :string
+  # Session name to use when assuming an IAM role
+  config :s3_role_session_name, :validate => :string, :default => "logstash"
 
 
   attr_reader :poller
@@ -148,13 +154,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     aws_sqs_client = Aws::SQS::Client.new(aws_options_hash)
     queue_url = aws_sqs_client.get_queue_url({ queue_name: @queue, queue_owner_aws_account_id: @queue_owner_aws_account_id})[:queue_url]
     @poller = Aws::SQS::QueuePoller.new(queue_url, :client => aws_sqs_client)
-    if s3_access_key_id and s3_secret_access_key
-      @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key) )
-      @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
-    else
-      @s3_client = Aws::S3::Client.new(aws_options_hash)
-    end
-
+    get_s3client
     @s3_resource = get_s3object
   rescue Aws::SQS::Errors::ServiceError => e
     @logger.error("Cannot establish connection to Amazon SQS", :error => e)
@@ -277,24 +277,22 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
         # The line need to go through the codecs to replace
         # unknown bytes in the log stream before doing a regexp match or
         # you will get a `Error: invalid byte sequence in UTF-8'
-        local_decorate(event, key, folder, metadata, bucket)
-        queue << event
+        local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
       end
     end
     @logger.debug("end if file #{filename}")
     #@logger.info("event pre flush", :event => event)
     # #ensure any stateful codecs (such as multi-line ) are flushed to the queue
     instance_codec.flush do |event|
-      local_decorate(event, key, folder, metadata, bucket)
+      local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
       @logger.debug("We´e to flush an incomplete event...", :event => event)
-      queue << event
     end
 
     return true
   end # def process_local_log
 
   private
-  def local_decorate(event, key, folder, metadata, bucket)
+  def local_decorate_and_queue(event, queue, key, folder, metadata, bucket)
     if event_is_metadata?(event)
       @logger.debug('Event is metadata, updating the current cloudfront metadata', :event => event)
       update_metadata(metadata, event)
@@ -308,6 +306,7 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
       event.set("[@metadata][s3]", { "object_key"    => key })
       event.set("[@metadata][s3]", { "bucket_name"   => bucket })
       event.set("[@metadata][s3]", { "object_folder" => folder})
+      queue << event
     end
   end
 
@@ -367,11 +366,34 @@ class LogStash::Inputs::S3SNSSQS < LogStash::Inputs::Threadable
     end
   end
 
+
+  private
+  def get_s3client
+    if s3_access_key_id and s3_secret_access_key
+      @logger.debug("Using S3 Credentials from config", :ID => aws_options_hash.merge(:access_key_id => s3_access_key_id) )
+      @s3_client = Aws::S3::Client.new(aws_options_hash.merge(:access_key_id => s3_access_key_id, :secret_access_key => s3_secret_access_key))
+    elsif @s3_role_arn
+      @s3_client = Aws::S3::Client.new(aws_options_hash.merge!({ :credentials => s3_assume_role }))
+      @logger.debug("Using S3 Credentials from role", :s3client => @s3_client.inspect, :options => aws_options_hash.merge!({ :credentials => s3_assume_role }))
+    else
+      @s3_client = Aws::S3::Client.new(aws_options_hash)
+    end
+  end
+
   private
   def get_s3object
     s3 = Aws::S3::Resource.new(client: @s3_client)
   end
 
+  private
+  def s3_assume_role()
+    Aws::AssumeRoleCredentials.new(
+        client: Aws::STS::Client.new(region: @region),
+        role_arn: @s3_role_arn,
+        role_session_name: @s3_role_session_name
+    )
+  end
+
   private
   def event_is_metadata?(event)
     return false unless event.get("message").class == String

data/logstash-input-s3-sns-sqs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-input-s3-sns-sqs'
-  s.version         = '1.4.6'
+  s.version         = '1.4.8'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Get logs from AWS s3 buckets as issued by an object-created event via sns -> sqs."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3-sns-sqs
 version: !ruby/object:Gem::Version
-  version: 1.4.6
+  version: 1.4.8
 platform: ruby
 authors:
 - Christian Herweg
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-03-22 00:00:00.000000000 Z
+date: 2018-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement