logstash-input-s3 3.1.8 → 3.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4a526bec73e4d07a3547e1470e6733e45087d3b3a9904af4489df0e5f8735c4c
-  data.tar.gz: 80ff8f5c24cce9e91a8c848c2d31554188e6e685a2919d03122bb7eb95c5e024
+  metadata.gz: 60a19b3b0dff622aedd2d5dbbc373af2e7d778d0dadc50cbbccfb374ad81bca5
+  data.tar.gz: dcf91569049bb072699ec4134429bef539ccdd593a4096c99a944c2ea1b903b9
 SHA512:
-  metadata.gz: 5d14998fbcbf09289e6c0fde186d2c56154dad02196b3e222ee5b85c959122b6e4d8dae4861676016b801b8ba2f9bbdf8341b7d4cad86d43d60bfacf9d50c548
-  data.tar.gz: 16753788a73d9df21feb9bbb9ee82f8791c4a477563f0ffd44055a15e2463d654bd629815f20c30c0a5d680db79a240d6344713b584d6f11f2e7a87a73b489d0
+  metadata.gz: 8d4f33328b99e69f45d69740b434d30c8eea1e3ffe9d07faf021c350447520b3a5d227c99fdfbe7f52ddbd8812ae1c4d44605a08e44f3c1dbf20cca9ff756143
+  data.tar.gz: 6eaa8fa809d8b8ecb9390cf148b964edadca7aee861f94d0f4cb62c694f9d1a6174a19025c4286dcb1f694d3395bbd3013a432486009749c49d5d2961f284734

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 3.1.9
+  - Change default sincedb path to live in `{path.data}/plugins/inputs/s3` instead of $HOME.
+    Prior Logstash installations (using $HOME default) are automatically migrated.
+  - Don't download the file if the length is 0 #2
+
 ## 3.1.8
   - Update gemspec summary
 

data/docs/index.asciidoc

@@ -196,8 +196,9 @@ The AWS Session token for temporary credential
 
 Where to write the since database (keeps track of the date
 the last handled file was added to S3). The default will write
-sincedb files to some path matching "$HOME/.sincedb*"
-Should be a path with filename not just a directory.
+sincedb files to in the directory '{path.data}/plugins/inputs/s3/'
+
+If specified, this setting must be a filename path and not just a directory.
 
 [id="plugins-{type}s-{plugin}-temporary_directory"]
 ===== `temporary_directory` 
@@ -211,4 +212,4 @@ default to the current OS temporary directory in linux /tmp/logstash
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/inputs/s3.rb

@@ -27,9 +27,11 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # If specified, the prefix of filenames in the bucket must match (not a regexp)
   config :prefix, :validate => :string, :default => nil
 
-  # Where to write the since database (keeps track of the date
-  # the last handled file was added to S3). The default will write
-  # sincedb files to some path matching "$HOME/.sincedb*"
+  # The path to use for writing state. The state stored by this plugin is
+  # a memory of files already processed by this plugin.
+  #
+  # If not specified, the default is in `{path.data}/plugins/inputs/s3/...`
+  #
   # Should be a path with filename not just a directory.
   config :sincedb_path, :validate => :string, :default => nil
 
@@ -102,7 +104,7 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
       @logger.debug("S3 input: Found key", :key => log.key)
 
       unless ignore_filename?(log.key)
-        if sincedb.newer?(log.last_modified)
+        if sincedb.newer?(log.last_modified) && log.content_length > 0
           objects[log.key] = log.last_modified
           @logger.debug("S3 input: Adding to objects[]", :key => log.key)
           @logger.debug("objects[] length is: ", :length => objects.length)
@@ -286,7 +288,29 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
 
   private
   def sincedb_file
-    File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
+    digest = Digest::MD5.hexdigest("#{@bucket}+#{@prefix}")
+    dir = File.join(LogStash::SETTINGS.get_value("path.data"), "plugins", "inputs", "s3")
+    FileUtils::mkdir_p(dir)
+    path = File.join(dir, "sincedb_#{digest}")
+
+    # Migrate old default sincedb path to new one.
+    if ENV["HOME"]
+      # This is the old file path including the old digest mechanism.
+      # It remains as a way to automatically upgrade users with the old default ($HOME)
+      # to the new default (path.data)
+      old = File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
+      if File.exist?(old)
+        logger.info("Migrating old sincedb in $HOME to {path.data}")
+        FileUtils.mv(old, path)
+      end
+    end
+
+    path
+  end
+
+
+  private
+  def old_sincedb_file
   end
 
   private

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '3.1.8'
+  s.version         = '3.1.9'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Streams events from files in a S3 bucket"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -20,7 +20,7 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 2.1.12", "<= 2.99"
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '~> 0.0.18'
  # s.add_runtime_dependency 'aws-sdk-resources', '>= 2.0.33'

data/spec/inputs/s3_spec.rb

@@ -81,11 +81,11 @@ describe LogStash::Inputs::S3 do
   describe "#list_new_files" do
     before { allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects_list } }
 
-    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now) }
+    let!(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now, :content_length => 10) }
     let(:objects_list) {
       [
-        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day),
-        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day),
+        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day, :content_length => 100),
+        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day, :content_length => 50),
         present_object
       ]
     }
@@ -105,7 +105,7 @@ describe LogStash::Inputs::S3 do
     context "If the bucket is the same as the backup bucket" do
       it 'should ignore files from the bucket if they match the backup prefix' do
         objects_list = [
-          double(:key => 'mybackup-log-1', :last_modified => Time.now),
+          double(:key => 'mybackup-log-1', :last_modified => Time.now, :content_length => 5),
           present_object
         ]
 
@@ -131,7 +131,7 @@ describe LogStash::Inputs::S3 do
         prefix = 'mysource/'
 
         objects_list = [
-          double(:key => prefix, :last_modified => Time.now),
+          double(:key => prefix, :last_modified => Time.now, :content_length => 5),
           present_object
         ]
 
@@ -144,9 +144,9 @@ describe LogStash::Inputs::S3 do
 
     it 'should sort return object sorted by last_modification date with older first' do
       objects = [
-        double(:key => 'YESTERDAY', :last_modified => Time.now - day),
-        double(:key => 'TODAY', :last_modified => Time.now),
-        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day)
+        double(:key => 'YESTERDAY', :last_modified => Time.now - day, :content_length => 5),
+        double(:key => 'TODAY', :last_modified => Time.now, :content_length => 5),
+        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day, :content_length => 5)
       ]
 
       allow_any_instance_of(Aws::S3::Bucket).to receive(:objects) { objects }
@@ -225,7 +225,7 @@ describe LogStash::Inputs::S3 do
 
   context 'when working with logs' do
     let(:objects) { [log] }
-    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day) }
+    let(:log) { double(:key => 'uncompressed.log', :last_modified => Time.now - 2 * day, :content_length => 5) }
     let(:data) { File.read(log_file) }
 
     before do
@@ -270,7 +270,7 @@ describe LogStash::Inputs::S3 do
     end
 
     context "multiple compressed streams" do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiple_compressed_streams.gz') }
 
       include_examples "generated events" do
@@ -279,7 +279,7 @@ describe LogStash::Inputs::S3 do
     end
       
     context 'compressed' do
-      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day) }
+      let(:log) { double(:key => 'log.gz', :last_modified => Time.now - 2 * day, :content_length => 5) }
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'compressed.log.gz') }
 
       include_examples "generated events"

metadata

@@ -1,21 +1,21 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 3.1.8
+  version: 3.1.9
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2017-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 2.1.12
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 2.1.12
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Streams events from files in a S3 bucket