logstash-input-s3 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 18491f4d5a8850a30b8bd5b25c33cc2a0dd9c247
-  data.tar.gz: c3017df48dd6ea484f8afb81e77d5b1e085d9d11
+  metadata.gz: 1be6737259a6ffb9f0f0734931ab2f495912bac4
+  data.tar.gz: f794cf9ce4f876bb3ecd81ab0f167cefc1fd692d
 SHA512:
-  metadata.gz: 8d112c0f09cbfe563eebce342812e8082b125b660560a250971b81a16d12584cad7303cc2b8b37fd2c6dfb2d64a0b44b44b843e133a22cde89971b1568bc38f8
-  data.tar.gz: ecec7bab79976f09f84f72b516ebe95dfbc6e07998f23ff581e17e275ab994259707e311aed5441bb38be5557bbed8ca377abfeb41f701fdcd8e79716954a51e
+  metadata.gz: 97dbbbda141e669cdad3bfce516764e649ccf929d72509d858e86211f772fb8f2e91311b945ff7576616a941f161afe41faa57769913b83e72b9697a6e073140
+  data.tar.gz: d6235bf2a17418da6108349144dfef750526eaa2405a09607e49db1748e76f88f5e14851a6384fbe06a4ae26cde4d7de2187a069aba3b6664c9c2e9fe3c25ef7

data/lib/logstash/inputs/s3.rb

@@ -1,44 +1,41 @@
 # encoding: utf-8
 require "logstash/inputs/base"
 require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
 
 require "time"
 require "tmpdir"
+require "stud/interval"
+require "stud/temporary"
 
 # Stream events from files from a S3 bucket.
 #
 # Each line from each file generates an event.
 # Files ending in `.gz` are handled as gzip'ed files.
 class LogStash::Inputs::S3 < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig
+
   config_name "s3"
   milestone 1
 
-  # TODO(sissel): refactor to use `line` codec (requires removing both gzip
-  # support and readline usage). Support gzip through a gzip codec! ;)
-  default :codec, "plain"
+  default :codec, "line"
 
-  # The credentials of the AWS account used to access the bucket.
+  # DEPRECATED: The credentials of the AWS account used to access the bucket.
   # Credentials can be specified:
-  # - As an `["id","secret"]` array
-  # - As a path to a file containing `AWS_ACCESS_KEY_ID=...` and `AWS_SECRET_ACCESS_KEY=...`
-  # - In the environment, if not set (using variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`)
-  config :credentials, :validate => :array, :default => []
+  # - As an ["id","secret"] array
+  # - As a path to a file containing AWS_ACCESS_KEY_ID=... and AWS_SECRET_ACCESS_KEY=...
+  # - In the environment, if not set (using variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY)
+  config :credentials, :validate => :array, :default => [], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
 
   # The name of the S3 bucket.
   config :bucket, :validate => :string, :required => true
 
-  # The AWS region for your bucket.
-  config :region, :validate => ["us-east-1", "us-west-1", "us-west-2",
-                                "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"],
-                                :deprecated => "'region' has been deprecated in favor of 'region_endpoint'"
-
   # The AWS region for your bucket.
   config :region_endpoint, :validate => ["us-east-1", "us-west-1", "us-west-2",
                                 "eu-west-1", "ap-southeast-1", "ap-southeast-2",
-                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => "us-east-1"
+                                "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :deprecated => "This only exists to be backwards compatible. This plugin now uses the AwsConfig from PluginMixins"
 
-  # If specified, the prefix the filenames in the bucket must match (not a regexp)
+  # If specified, the prefix of filenames in the bucket must match (not a regexp)
   config :prefix, :validate => :string, :default => nil
 
   # Where to write the since database (keeps track of the date
@@ -50,6 +47,11 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # Name of a S3 bucket to backup processed files to.
   config :backup_to_bucket, :validate => :string, :default => nil
 
+  # Append a prefix to the key (full path including file name in s3) after processing.
+  # If backing up to another (or the same) bucket, this effectively lets you
+  # choose a new 'folder' to place the files in
+  config :backup_add_prefix, :validate => :string, :default => nil
+
   # Path of a local directory to backup processed files to.
   config :backup_to_dir, :validate => :string, :default => nil
 
@@ -60,61 +62,19 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
   # Value is in seconds.
   config :interval, :validate => :number, :default => 60
 
+  # Ruby style regexp of keys to exclude from the bucket
+  config :exclude_pattern, :validate => :string, :default => nil
+
   public
   def register
     require "digest/md5"
     require "aws-sdk"
 
-    @region_endpoint = @region if @region && !@region.empty?
-
-    @logger.info("Registering s3 input", :bucket => @bucket, :region_endpoint => @region_endpoint)
-
-    if @credentials.length == 0
-      @access_key_id = ENV['AWS_ACCESS_KEY_ID']
-      @secret_access_key = ENV['AWS_SECRET_ACCESS_KEY']
-    elsif @credentials.length == 1
-      File.open(@credentials[0]) { |f| f.each do |line|
-        unless (/^\#/.match(line))
-          if(/\s*=\s*/.match(line))
-            param, value = line.split('=', 2)
-            param = param.chomp().strip()
-            value = value.chomp().strip()
-            if param.eql?('AWS_ACCESS_KEY_ID')
-              @access_key_id = value
-            elsif param.eql?('AWS_SECRET_ACCESS_KEY')
-              @secret_access_key = value
-            end
-          end
-        end
-      end
-      }
-    elsif @credentials.length == 2
-      @access_key_id = @credentials[0]
-      @secret_access_key = @credentials[1]
-    else
-      raise ArgumentError.new('Credentials must be of the form "/path/to/file" or ["id", "secret"]')
-    end
-
-    if @access_key_id.nil? or @secret_access_key.nil?
-      raise ArgumentError.new('Missing AWS credentials')
-    end
-
-    if @bucket.nil?
-      raise ArgumentError.new('Missing AWS bucket')
-    end
+    @region = get_region
 
-    if @sincedb_path.nil?
-      if ENV['HOME'].nil?
-        raise ArgumentError.new('No HOME or sincedb_path set')
-      end
-      @sincedb_path = File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
-    end
+    @logger.info("Registering s3 input", :bucket => @bucket, :region => @region)
 
-    s3 = AWS::S3.new(
-      :access_key_id => @access_key_id,
-      :secret_access_key => @secret_access_key,
-      :region => @region_endpoint
-    )
+    s3 = get_s3object
 
     @s3bucket = s3.buckets[@bucket]
 
@@ -128,152 +88,212 @@ class LogStash::Inputs::S3 < LogStash::Inputs::Base
     unless @backup_to_dir.nil?
       Dir.mkdir(@backup_to_dir, 0700) unless File.exists?(@backup_to_dir)
     end
-
   end # def register
 
+
   public
   def run(queue)
-    loop do
-      process_new(queue)
-      sleep(@interval)
+    Stud.interval(@interval) do
+      process_files(queue)
     end
-    finished
   end # def run
 
-  private
-  def process_new(queue, since=nil)
+  public
+  def list_new_files
+    objects = {}
 
-    if since.nil?
-        since = sincedb_read()
-    end
+    @s3bucket.objects.with_prefix(@prefix).each do |log|
+      @logger.debug("S3 input: Found key", :key => log.key)
 
-    objects = list_new(since)
-    objects.each do |k|
-      @logger.debug("S3 input processing", :bucket => @bucket, :key => k)
-      lastmod = @s3bucket.objects[k].last_modified
-      process_log(queue, k)
-      sincedb_write(lastmod)
+      unless ignore_filename?(log.key)
+        if sincedb.newer?(log.last_modified)
+          objects[log.key] = log.last_modified
+          @logger.debug("S3 input: Adding to objects[]", :key => log.key)
+        end
+      end
     end
+    return objects.keys.sort {|a,b| objects[a] <=> objects[b]}
+  end # def fetch_new_files
 
-  end # def process_new
 
-  private
-  def list_new(since=nil)
-
-    if since.nil?
-      since = Time.new(0)
+  public
+  def backup_to_bucket(object, key)
+    unless @backup_to_bucket.nil?
+      backup_key = "#{@backup_add_prefix}#{key}"
+      if @delete
+        object.move_to(backup_key, :bucket => @backup_bucket)
+      else
+        object.copy_to(backup_key, :bucket => @backup_bucket)
+      end
     end
+  end
 
-    objects = {}
-    @s3bucket.objects.with_prefix(@prefix).each do |log|
-      if log.last_modified > since
-        objects[log.key] = log.last_modified
-      end
+  public
+  def backup_to_dir(filename)
+    unless @backup_to_dir.nil?
+      FileUtils.cp(filename, @backup_to_dir)
     end
+  end
 
-    return sorted_objects = objects.keys.sort {|a,b| objects[a] <=> objects[b]}
+  private
+  def process_local_log(queue, filename)
+    @codec.decode(File.open(filename, 'rb')) do |event|
+      decorate(event)
+      queue << event
+    end
+  end # def process_local_log
+  
+  private
+  def sincedb 
+    @sincedb ||= if @sincedb_path.nil?
+                    @logger.info("Using default generated file for the sincedb", :filename => sincedb_file)
+                    SinceDB::File.new(sincedb_file)
+                  else
+                    @logger.error("S3 input: Configuration error, no HOME or sincedb_path set")
+                    SinceDB::File.new(@sincedb_path)
+                  end
+  end
 
-  end # def list_new
+  private
+  def sincedb_file
+    File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@bucket}+#{@prefix}"))
+  end
 
   private
-  def process_log(queue, key)
+  def process_files(queue, since=nil)
+    objects = list_new_files
+    objects.each do |key|
+      @logger.debug("S3 input processing", :bucket => @bucket, :key => key)
 
-    object = @s3bucket.objects[key]
-    tmp = Dir.mktmpdir("logstash-")
-    begin
-      filename = File.join(tmp, File.basename(key))
-      File.open(filename, 'wb') do |s3file|
-        object.read do |chunk|
-          s3file.write(chunk)
-        end
-      end
-      process_local_log(queue, filename)
-      unless @backup_to_bucket.nil?
-        backup_object = @backup_bucket.objects[key]
-        backup_object.write(Pathname.new(filename))
-      end
-      unless @backup_to_dir.nil?
-        FileUtils.cp(filename, @backup_to_dir)
-      end
-      if @delete
-        object.delete()
-      end
+      lastmod = @s3bucket.objects[key].last_modified
+
+      process_log(queue, key)
+
+      sincedb.write(lastmod)
     end
-    FileUtils.remove_entry_secure(tmp, force=true)
+  end # def process_files
 
-  end # def process_log
+  private
+  def ignore_filename?(filename)
+    if (@backup_add_prefix && @backup_to_bucket == @bucket && filename =~ /^#{backup_add_prefix}/)
+      return true
+    elsif @exclude_pattern.nil?
+      return false
+    elsif filename =~ Regexp.new(@exclude_pattern)
+      return true
+    else
+      return false
+    end
+  end
 
   private
-  def process_local_log(queue, filename)
+  def process_log(queue, key)
+    object = @s3bucket.objects[key]
 
-    metadata = {
-      :version => nil,
-      :format => nil,
-    }
-    File.open(filename) do |file|
-      if filename.end_with?('.gz')
-        gz = Zlib::GzipReader.new(file)
-        gz.each_line do |line|
-          metadata = process_line(queue, metadata, line)
-        end
-      else
-        file.each do |line|
-          metadata = process_line(queue, metadata, line)
-        end
+    tmp = Stud::Temporary.directory("logstash-")
+
+    filename = File.join(tmp, File.basename(key))
+
+    download_remote_file(object, filename)
+
+    process_local_log(queue, filename)
+
+    backup_to_bucket(object, key)
+    backup_to_dir(filename)
+
+    delete_file_from_bucket(object)
+  end
+
+  private
+  def download_remote_file(remote_object, local_filename)
+    @logger.debug("S3 input: Download remove file", :remote_key => remote_object.key, :local_filename => local_filename)
+    File.open(local_filename, 'wb') do |s3file|
+      remote_object.read do |chunk|
+        s3file.write(chunk)
       end
     end
-
-  end # def process_local_log
+  end
 
   private
-  def process_line(queue, metadata, line)
+  def delete_file_from_bucket(object)
+    if @delete and @backup_to_bucket.nil?
+      object.delete()
+    end
+  end
 
-    if /#Version: .+/.match(line)
-      junk, version = line.strip().split(/#Version: (.+)/)
-      unless version.nil?
-        metadata[:version] = version
-      end
-    elsif /#Fields: .+/.match(line)
-      junk, format = line.strip().split(/#Fields: (.+)/)
-      unless format.nil?
-        metadata[:format] = format
-      end
+  private
+  def get_region
+    # TODO: (ph) Deprecated, it will be removed
+    if @region_endpoint
+      @region_endpoint
     else
-      @codec.decode(line) do |event|
-        decorate(event)
-        unless metadata[:version].nil?
-          event["cloudfront_version"] = metadata[:version]
-        end
-        unless metadata[:format].nil?
-          event["cloudfront_fields"] = metadata[:format]
-        end
-        queue << event
-      end
+      @region
     end
-    return metadata
-
-  end # def process_line
+  end
 
   private
-  def sincedb_read()
+  def get_s3object
+    # TODO: (ph) Deprecated, it will be removed
+    if @credentials.length == 1
+      File.open(@credentials[0]) { |f| f.each do |line|
+        unless (/^\#/.match(line))
+          if(/\s*=\s*/.match(line))
+            param, value = line.split('=', 2)
+            param = param.chomp().strip()
+            value = value.chomp().strip()
+            if param.eql?('AWS_ACCESS_KEY_ID')
+              @access_key_id = value
+            elsif param.eql?('AWS_SECRET_ACCESS_KEY')
+              @secret_access_key = value
+            end
+          end
+        end
+      end
+      }
+    elsif @credentials.length == 2
+      @access_key_id = @credentials[0]
+      @secret_access_key = @credentials[1]
+    end
 
-    if File.exists?(@sincedb_path)
-      since = Time.parse(File.read(@sincedb_path).chomp.strip)
+    if @credentials
+      s3 = AWS::S3.new(
+        :access_key_id => @access_key_id,
+        :secret_access_key => @secret_access_key,
+        :region => @region
+      )
     else
-      since = Time.new(0)
+      s3 = AWS::S3.new(aws_options_hash)
     end
-    return since
-
-  end # def sincedb_read
+  end
 
   private
-  def sincedb_write(since=nil)
+  def aws_service_endpoint(region)
+    return { :s3_endpoint => region }
+  end
+
+  module SinceDB
+    class File
+      def initialize(file)
+        @sincedb_path = file
+      end
 
-    if since.nil?
-      since = Time.now()
-    end
-    File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
+      def newer?(date)
+        date > read
+      end
 
-  end # def sincedb_write
+      def read
+        if ::File.exists?(@sincedb_path)
+          since = Time.parse(::File.read(@sincedb_path).chomp.strip)
+        else
+          since = Time.new(0)
+        end
+        return since
+      end
 
+      def write(since = nil)
+        since = Time.now() if since.nil?
+        ::File.open(@sincedb_path, 'w') { |file| file.write(since.to_s) }
+      end
+    end
+  end
 end # class LogStash::Inputs::S3

data/logstash-input-s3.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-input-s3'
-  s.version         = '0.1.1'
+  s.version         = '0.1.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Stream events from files from a S3 bucket."
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -22,8 +22,9 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
   s.add_runtime_dependency 'logstash-mixin-aws'
-  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-codec-line'
   s.add_runtime_dependency 'aws-sdk'
+  s.add_runtime_dependency 'stud', '~> 0.0.18'
 
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/inputs/s3_spec.rb

@@ -1,5 +1,146 @@
+# encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/s3"
+require "logstash/errors"
 require 'logstash/inputs/s3'
 
+require "aws-sdk"
+require "stud/temporary"
+
 describe LogStash::Inputs::S3 do
+  before do
+    AWS.stub!
+    Thread.abort_on_exception = true
+  end
+  let(:day) { 3600 * 24 }
+  let(:settings) {
+    {
+      "access_key_id" => "1234",
+      "secret_access_key" => "secret",
+      "bucket" => "logstash-test"
+    }
+  }
+
+  describe "#list_new_files" do
+    before { allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list } }
+
+    let(:present_object) { double(:key => 'this-should-be-present', :last_modified => Time.now) }
+    let(:objects_list) {
+      [
+        double(:key => 'exclude-this-file-1', :last_modified => Time.now - 2 * day),
+        double(:key => 'exclude/logstash', :last_modified => Time.now - 2 * day),
+        present_object
+      ]
+    }
+
+    it 'should allow user to exclude files from the s3 bucket' do
+      config = LogStash::Inputs::S3.new(settings.merge({ "exclude_pattern" => "^exclude" }))
+      config.register
+      expect(config.list_new_files).to eq([present_object.key])
+    end
+
+    it 'should support not providing a exclude pattern' do
+      config = LogStash::Inputs::S3.new(settings)
+      config.register
+      expect(config.list_new_files).to eq(objects_list.map(&:key))
+    end
+
+    context "If the bucket is the same as the backup bucket" do
+      it 'should ignore files from the bucket if they match the backup prefix' do
+        objects_list = [
+          double(:key => 'mybackup-log-1', :last_modified => Time.now),
+          present_object
+        ]
+
+        allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects_list }
+
+        config = LogStash::Inputs::S3.new(settings.merge({ 'backup_add_prefix' => 'mybackup',
+                                                           'backup_to_bucket' => settings['bucket']}))
+        config.register
+        expect(config.list_new_files).to eq([present_object.key])
+      end
+    end
+
+    it 'should ignore files older than X' do
+      config = LogStash::Inputs::S3.new(settings.merge({ 'backup_add_prefix' => 'exclude-this-file'}))
+
+      expect_any_instance_of(LogStash::Inputs::S3::SinceDB::File).to receive(:read).exactly(objects_list.size) { Time.now - day }
+      config.register
+
+      expect(config.list_new_files).to eq([present_object.key])
+    end
+
+    it 'should sort return object sorted by last_modification date with older first' do
+      objects = [
+        double(:key => 'YESTERDAY', :last_modified => Time.now - day),
+        double(:key => 'TODAY', :last_modified => Time.now),
+        double(:key => 'TWO_DAYS_AGO', :last_modified => Time.now - 2 * day)
+      ]
+
+      allow_any_instance_of(AWS::S3::ObjectCollection).to receive(:with_prefix).with(nil) { objects }
+
+
+      config = LogStash::Inputs::S3.new(settings)
+      config.register
+      expect(config.list_new_files).to eq(['TWO_DAYS_AGO', 'YESTERDAY', 'TODAY'])
+    end
+
+    describe "when doing backup on the s3" do
+      it 'should copy to another s3 bucket when keeping the original file' do
+        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup"}))
+        config.register
+
+        s3object = double()
+        expect(s3object).to receive(:copy_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+
+        config.backup_to_bucket(s3object, 'test-file')
+      end
+
+      it 'should move to another s3 bucket when deleting the original file' do
+        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup", "delete" => true }))
+        config.register
+
+        s3object = double()
+        expect(s3object).to receive(:move_to).with('test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+
+        config.backup_to_bucket(s3object, 'test-file')
+      end
+
+      it 'should add the specified prefix to the backup file' do
+        config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_bucket" => "mybackup",
+                                                           "backup_add_prefix" => 'backup-' }))
+        config.register
+
+        s3object = double()
+        expect(s3object).to receive(:copy_to).with('backup-test-file', :bucket => an_instance_of(AWS::S3::Bucket))
+
+        config.backup_to_bucket(s3object, 'test-file')
+      end
+    end
+
+    it 'should support doing local backup of files' do
+      Stud::Temporary.directory do |backup_dir|
+        Stud::Temporary.file do |source_file|
+          backup_file = File.join(backup_dir.to_s, Pathname.new(source_file.path).basename.to_s)
+
+          config = LogStash::Inputs::S3.new(settings.merge({ "backup_to_dir" => backup_dir }))
+
+          config.backup_to_dir(source_file)
+
+          expect(File.exists?(backup_file)).to be_true
+        end
+      end
+    end
+
+    it 'should accepts a list of credentials for the aws-sdk, this is deprecated' do
+      old_credentials_settings = {
+        "credentials" => ['1234', 'secret'],
+        "backup_to_dir" => "/tmp/mybackup",
+        "bucket" => "logstash-test"
+      }
+
+      config = LogStash::Inputs::S3.new(old_credentials_settings)
+      expect{ config.register }.not_to raise_error
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-11-19 00:00:00.000000000 Z
+date: 2015-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash
@@ -45,7 +45,7 @@ dependencies:
   prerelease: false
   type: :runtime
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
+  name: logstash-codec-line
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -72,6 +72,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: stud
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.0.18
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
@@ -121,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.2.2
+rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Stream events from files from a S3 bucket.