logstash-input-multirds 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e397a301602ab70c6241f3e12f97f0b748868ca3
+  data.tar.gz: fb55e7f3bccd472bbef7a8fc3a02b1277bf812d6
+SHA512:
+  metadata.gz: 3ae26cdbf5cbab8f1a863a7cce692351b7775b1af471360c3df2e808c85a8b9c4828deaff09d4caca909e375094f9999533819b1f9427ecf990668e50ffb5f61
+  data.tar.gz: bc3d45b3f9fad096079233c7c766e66ff80060b1993dfb2f6f0361f11413404b083d6d6c2a9c496e95bc25debfc067671c783c2c8c02de92fcd6da614e4ea67a

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,11 @@
+# Logstash Input Multi-RDS
+
+Forked from discourse/logstash-input-rds I needed competing consumer and multi-db support 
+    input {
+      rds {
+        region => "us-west-2"
+        instance_name_pattern => ".*"
+        log_file_name_pattern => ".*"
+        group_name => "rds"
+      }
+    }

data/lib/logstash/inputs/multirds/patch.rb

@@ -0,0 +1,12 @@
+require 'aws-sdk'
+
+begin
+  old_stderr = $stderr
+  $stderr = StringIO.new
+
+  module Aws
+    const_set(:RDS, Aws::RDS)
+  end
+ensure
+  $stderr = old_stderr
+end

data/lib/logstash/inputs/multirds.rb

@@ -0,0 +1,100 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require "aws-sdk"
+require "logstash/inputs/rds/patch"
+require "logstash/plugin_mixins/aws_config"
+require "time"
+
+Aws.eager_autoload!
+
+class LogStash::Inputs::Multirds < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+
+  config_name "multirds"
+  milestone 1
+  default :codec, "plain"
+
+  config :instance_name_pattern, :validate => :string, :required => true
+  config :log_file_name_pattern, :validate => :string, :required => true
+  config :polling_frequency, :validate => :number, :default => 600
+  config :group_name, :validate => :string, :required => true
+  
+  def register
+    # @logger.info "Registering multi-RDS input", :region => @region, :instance => @instance_name, :log_file => @log_file_name
+    # @database = Aws::RDS::DBInstance.new @instance_name, aws_options_hash
+    # path = @sincedb_path || File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@instance_name}+#{@log_file_name}"))
+    # @sincedb = SinceDB::File.new path
+    @logger.info "Registering multi-rds input", :instance_name_pattern => @instance_name_pattern, :log_file_name_pattern => @log_file_name_pattern, :group_name = @group_name
+    @db = Aws::DynamoDB::Client.new
+    @rds = Aws::RDS::Client.new
+    # TODO: Auto-create dynamodb table here -- should that be a param? 
+  end
+
+  def run(queue)
+    @thread = Thread.current
+    Stud.interval(@polling_frequency) do
+      @logger.debug "finding #{@log_file_name} for #{@instance_name} starting #{@sincedb.read} (#{@sincedb.read.to_i * 1000})"
+      begin
+        logfiles = @database.log_files({
+          filename_contains: @log_file_name,
+          file_last_written: @sincedb.read.to_i * 1000,
+        })
+        logfiles.each do |logfile|
+          @logger.debug "downloading #{logfile.name} for #{@instance_name}"
+          more = true
+          marker = "0"
+          while more do
+            response = logfile.download({marker: marker})
+            response[:log_file_data].lines.each do |line|
+              @codec.decode(line) do |event|
+                decorate event
+                event.set "rds_instance", @instance_name
+                event.set "log_file", @log_file_name
+                queue << event
+              end
+            end
+            more = response[:additional_data_pending]
+            marker = response[:marker]
+          end
+          @sincedb.write (filename2datetime logfile.name)
+        end
+      rescue Aws::RDS::Errors::ServiceError
+        # the next iteration will resume at the same location
+        @logger.warn "caught AWS service error"
+      end
+    end
+  end
+
+  def stop
+    Stud.stop! @thread
+  end
+
+  def filename2datetime(name)
+    parts = name.match /(\d{4})-(\d{2})-(\d{2})-(\d{2})$/
+    Time.utc parts[1], parts[2], parts[3], parts[4]
+  end
+
+  private
+  module SinceDB
+    class File
+      def initialize(file)
+        @db = file
+      end
+
+      def read
+        if ::File.exists?(@db)
+          content = ::File.read(@db).chomp.strip
+          return content.empty? ? Time.new : Time.parse(content)
+        else
+          return Time.new("1999-01-01")
+        end
+      end
+
+      def write(time)
+        ::File.open(@db, 'w') { |file| file.write time.to_s }
+      end
+    end
+  end
+end

data/logstash-input-multirds.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-multirds'
+  s.version       = '0.0.1'
+  s.summary       = 'Ingest RDS log files to Logstash with competing consumers and multiple databases'
+
+  s.authors       = ['Robert Labrie']
+  s.email         = ['robert.labrie@gmail.com']
+  s.homepage      = 'https://github.com/robertlabrie/logstash-input-multi-rds'
+
+  s.require_paths = ['lib']
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-mixin-aws'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+end

metadata

@@ -0,0 +1,121 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-multirds
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Robert Labrie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-aws
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+description: 
+email:
+- robert.labrie@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/multirds.rb
+- lib/logstash/inputs/multirds/patch.rb
+- logstash-input-multirds.gemspec
+homepage: https://github.com/robertlabrie/logstash-input-multi-rds
+licenses: []
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Ingest RDS log files to Logstash with competing consumers and multiple databases
+test_files: []