logstash-input-elasticache 0.2.10

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a7b5c422e124c1074ca7d4c24b0ad7c7f639bdc2ff21a5e97f98d895e86b8f08
+  data.tar.gz: 4f9b9dd48c6d2f2c31d21eb2cefca706f637785648accd9a5f9ea8fcf303768f
+SHA512:
+  metadata.gz: a6e3555c977c4fdc759e3895098714856c7a7591c393e2bf7467d68ed8e31e2ab96599c05bc29a2708234e4494e7b8572bc76b735fc6ac502036790a18e11bcb
+  data.tar.gz: 5a756c510129835b2c6271b612364c7d4d6a2618a4e9241b2d777abfcb3bf8bafcc5ac318f479b9cfa81591026e2f3631c31c8d21f9d7feae568e80958dfdf71

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE

@@ -0,0 +1,11 @@
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,10 @@
+# Logstash Input ElastiCache
+
+    input {
+      elasticache {
+        region => "us-west-2"
+        source_type => "replication-group"
+        source_name => "development"
+      }
+    }
+

data/lib/logstash/inputs/elasticache.rb

@@ -0,0 +1,85 @@
+# encoding: utf-8
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/plugin_mixins/aws_config"
+require "stud/interval"
+require "aws-sdk"
+require "date"
+
+Aws.eager_autoload!
+
+class LogStash::Inputs::ElastiCache < LogStash::Inputs::Base
+  include LogStash::PluginMixins::AwsConfig::V2
+
+  config_name "elasticache"
+  milestone 1
+  default :codec, "plain"
+
+  config :source_type, :validate => :string, :required => true
+  config :source_name, :validate => :string, :required => true
+  config :polling_frequency, :validate => :number, :default => 600
+
+  def register
+    require "aws-sdk"
+    @logger.info "Registering ElastiCache input", :region => @region, :source_type => @source_type, :source_name => @source_name
+    @elasticache = Aws::ElastiCache::Client.new aws_options_hash
+    @since = DateTime.now - 13 # FIXME sincedb
+  end
+
+  def run(queue)
+    @thread = Thread.current
+    Stud.interval(@polling_frequency) do
+      with_rescue_and_retry do
+        checkpoint = Time.now
+
+        more = true
+        marker = nil
+        while more do
+          response = @elasticache.describe_events({
+            source_identifier: @source_name,
+            source_type: @source_type,
+            start_time: @since,
+            end_time: checkpoint,
+            marker: marker,
+          })
+
+          response[:events].each { |item| enqueue item, queue }
+          more = response[:marker]
+          marker = response[:marker]
+        end
+        @since = checkpoint
+      end
+    end
+  end
+
+  def stop
+    Stud.stop! @thread
+  end
+
+  def with_rescue_and_retry
+    begin
+      failures ||= 0
+      yield
+    rescue Aws::ElastiCache::Errors::ServiceError
+      failures += 1
+      if failures <= 4
+        @logger.warn "service error for #{@source_type} #{@source_name}, delaying attempt ##{failures + 1}"
+        sleep (2 ** failures)
+        retry
+      else
+        @logger.error "pausing #{@source_type} #{@source_name} after five failures, will resume in #{@polling_frequency}s"
+      end
+    end
+  end
+
+  def enqueue(item, queue)
+    event = LogStash::Event.new({
+      "@timestamp" => item.date,
+      "message" => item.message,
+    })
+    decorate event
+    event.set @source_type.gsub(/-/, "_"), @source_name
+    @logger.debug "shipping #{event} to #{queue}"
+    queue << event
+  end
+end

data/logstash-input-elasticache.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name          = 'logstash-input-elasticache'
+  s.version       = '0.2.10'
+  s.summary       = 'Import events from ElastiCache to Logstash'
+
+  s.authors       = ['Andrew Schleifer']
+  s.email         = ['me@andrewschleifer.name']
+  s.homepage      = 'https://github.com/discourse/logstash-input-elasticache'
+
+  s.require_paths = ['lib']
+  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-mixin-aws'
+  s.add_runtime_dependency 'stud', '>= 0.0.22'
+  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-elasticache
+version: !ruby/object:Gem::Version
+  version: 0.2.10
+platform: ruby
+authors:
+- Andrew Schleifer
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-12-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-mixin-aws
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.16
+description:
+email:
+- me@andrewschleifer.name
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/elasticache.rb
+- logstash-input-elasticache.gemspec
+homepage: https://github.com/discourse/logstash-input-elasticache
+licenses: []
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.13
+signing_key:
+specification_version: 4
+summary: Import events from ElastiCache to Logstash
+test_files: []