logstash-input-rds 0.10.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f5d470b025d41bd47c4d16db6e024a57a7c22a85f0047b27037f79bc4fed9175
+  data.tar.gz: 33bdc3b8fa47f832755aa73881d1c2c04e198a0b66774590d4203cd21afc273f
+SHA512:
+  metadata.gz: 3db6fc440d15ad18d0f7bb34ca49cb6acc729da5559231b927870312decc4007e3bbe8ab70c99148c642b643b6f47a503a0073e2d72642d38259e74c9dc89ef8
+  data.tar.gz: 7f1bf5c19c52cbc47c890bc0a53ae4297e99a07491ed06950de91d6920fc31ddc0df162bb5a70ea73c7f8b84971eb97b10b03b06da4f6811dfa359a3c8679ede

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,4 @@
+# Logstash Input RDS
+
+Ingest RDS log files to Logstash.
+

data/lib/logstash/inputs/rds.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "stud/interval"
+require "logstash/inputs/rds/patch"
+require "time"
+
+Aws.eager_autoload!
+
+class LogStash::Inputs::Rds < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+
+  config_name "rds"
+  milestone 1
+  default :codec, "plain"
+
+  config :instance_identifier, :validate => :string, :required => true
+  config :log_file_name, :validate => :string, :required => true
+  config :polling_frequency, :validate => :number, :default => 600
+
+  def register
+    require "aws-sdk"
+    @logger.info "Registering RDS input", :instance_identifier => @instance_identifier
+    @database = Aws::RDS::DBInstance.new @instance_identifier, aws_options_hash
+    @sincedate = filename2datetime "1999-01-01-01" # FIXME
+  end
+
+  def run(queue)
+    Stud.interval(@polling_frequency) do
+      @logger.info "finding files starting #{@sincedate} (#{@sincedate.to_i * 1000})"
+
+      logfiles = @database.log_files({
+        filename_contains: @log_file_name,
+        file_last_written: @sincedate.to_i * 1000,
+      })
+      logfiles.each do |logfile|
+        @sincedate = filename2datetime logfile.name
+
+        more = true
+        marker = "0"
+        while more do
+          response = logfile.download({marker: marker})
+          response[:log_file_data].lines.each do |line|
+            @codec.decode(line) do |event|
+              event.set "rds-instance", @instance_identifier
+              event.set "log-file", @log_file_name
+              queue << event
+            end
+          end
+          more = response[:additional_data_pending]
+          marker = response[:marker]
+        end
+      end
+    end
+  end
+
+  def filename2datetime(name)
+    fragments = name.match /(\d{4})-(\d{2})-(\d{2})-(\d{2})$/
+    Time.parse "#{fragments[1]}-#{fragments[2]}-#{fragments[3]} #{fragments[4]}"
+  end
+end
+

data/lib/logstash/inputs/rds/patch.rb

@@ -0,0 +1,12 @@
+require 'aws-sdk'
+
+begin
+  old_stderr = $stderr
+  $stderr = StringIO.new
+
+  module Aws
+    const_set(:RDS, Aws::RDS)
+  end
+ensure
+  $stderr = old_stderr
+end

data/logstash-input-rds.gemspec

@@ -0,0 +1,21 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-rds'
+  s.version       = '0.10.1'
+  s.summary       = 'Stream events from RDS to Logstash'
+
+  s.authors       = ['Andrew Schleifer']
+  s.email         = 'me@andrewschleifer.name'
+
+  s.require_paths = ['lib']
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core-plugin-api", "~> 2.0"
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+end

data/spec/inputs/rds_spec.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/rds"
+
+describe LogStash::Inputs::Rds do
+
+  it_behaves_like "an interruptible input plugin" do
+    let(:config) { { "interval" => 100 } }
+  end
+
+end

metadata

@@ -0,0 +1,108 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-rds
+version: !ruby/object:Gem::Version
+  version: 0.10.1
+platform: ruby
+authors:
+- Andrew Schleifer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-12-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+description:
+email: me@andrewschleifer.name
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/rds.rb
+- lib/logstash/inputs/rds/patch.rb
+- logstash-input-rds.gemspec
+- spec/inputs/rds_spec.rb
+homepage:
+licenses: []
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Stream events from RDS to Logstash
+test_files:
+- spec/inputs/rds_spec.rb