logstash-input-multirds 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7dfd5f53cd59c0c7d8c89f79973a92bacc911b87
-  data.tar.gz: 184bb1f97be2b8d5896427d9ac4b5b92a585894b
+  metadata.gz: 93c7bd1bdecf5dcf2dfa932257933cdb60ba75aa
+  data.tar.gz: c8900191f3a7f8b48de91d4f1d3cd88670a70295
 SHA512:
-  metadata.gz: ef525b86cb6430c1e35776a1101c22d7f3355d36c95ca79f7dc3a839fd59223c9c025c76411bcfa591ca54a79dd19e18ece8107948de4099b10ad760273dedef
-  data.tar.gz: 6ef768857f2d767e864f42d8afadc513fb15d722dd565e891e9c9927893bcbca2c26213d5017be1b3ea79f3e696d7d8d8c43949c127fb560fedb56b7c734f1a9
+  metadata.gz: f95d31355e93ba96590b29d1559d5102373df4bcf1f07830e85260dbfc3580d208db6e8a84f51292f5642de56bb4809490c9c17144b45121cd9fed8574be1df2
+  data.tar.gz: a5a5a0327183bac93810f10b6180400c82846fdfe82f803deae5961431590921c0e088c1f07da861901dfd19944e40704e96ad1c239c6ee548bb22d19659a34f

data/README.md

@@ -1,6 +1,17 @@
 # Logstash Input Multi-RDS
 
-Inputs RDS logs because Postgres doesn't support cloudwatch. Forked from discourse/logstash-input-rds I needed competing consumer and multi-db support. Uses DynamoDB for distributed locking and marker tracking. The plugin will create the table automatically if you give it permission, otherwise you can create it yourself where the table name is `group_name` and the primary key is a string called `id`.
+RDS Postgres instances do not support cloudwatch, logs must be polled using the AWS API. This logstash input plugin polls RDS logs at a configured interval with support for multiple instances of logstash and multiple RDS databases (thus the name: multirds). Locking and marker tracking are done with DynamoDB. 
+
+The plugin will automatically create a DynamoDB table, but if you want to do it manually, the name must match the configured `group_name` and the primary key is `id`.
+
+## Special thanks
+
+* This is a fork of logstash input RDS (https://github.com/discourse/logstash-input-rds) though I think I made too many changes to ever merge it back in
+* Using DynamoDB for locking and marker tracking was stolen from the Kinesis input plugin (https://github.com/logstash-plugins/logstash-input-kinesis) (which we're also using for Cloudwatch logs) 
+* The lock code was taken from the Dynalock gem (https://github.com/tourlane/dynalock) which works fine, but I needed to store extra data on the lock record so I just copied their code
+
+## Configuration
+
 ```
     input {
       multirds {
@@ -13,8 +24,6 @@ Inputs RDS logs because Postgres doesn't support cloudwatch. Forked from discour
     }
 ```
 
-## Configuration
-
 * `region`: The AWS region for RDS. The AWS SDK reads this info from the usual places, so it's not required, but if you don't set it somewhere the plugin won't run
   * **required**: false
 

data/lib/logstash/inputs/multirds.rb

@@ -1,136 +1,154 @@
-# encoding: utf-8
-require "logstash/inputs/base"
-require "logstash/namespace"
-require "stud/interval"
-require "aws-sdk"
-require "logstash/inputs/multirds/patch"
-require "logstash/plugin_mixins/aws_config"
-require "time"
-require "socket"
+require 'logstash/inputs/base'
+require 'logstash/namespace'
+require 'stud/interval'
+require 'aws-sdk'
+require 'logstash/inputs/multirds/patch'
+require 'logstash/plugin_mixins/aws_config'
+require 'time'
+require 'socket'
 Aws.eager_autoload!
 
 class LogStash::Inputs::Multirds < LogStash::Inputs::Base
   include LogStash::PluginMixins::AwsConfig::V2
 
-  config_name "multirds"
+  config_name 'multirds'
   milestone 1
-  default :codec, "plain"
-
-  config :instance_name_pattern, :validate => :string, :default => '.*'
-  config :log_file_name_pattern, :validate => :string, :default => '.*'
-  config :polling_frequency, :validate => :number, :default => 600
-  config :group_name, :validate => :string, :required => true
-  config :client_id, :validate => :string
-  
+  default :codec, 'plain'
+
+  config :instance_name_pattern, validate: :string, default: '.*'
+  config :log_file_name_pattern, validate: :string, default: '.*'
+  config :polling_frequency, validate: :number, default: 600
+  config :group_name, validate: :string, required: true
+  config :client_id, validate: :string
+
   def ensure_lock_table(db, table)
     begin
-        tables = db.list_tables({
-
-        })
-        return true if tables.to_h[:table_names].to_a.include?(table)
-        # TODO: there is a potential race condition here where a table could come back in list_tables but not be in ACTIVE state we should check this better
-        result = db.create_table({
-            table_name: table,
-            key_schema: [
-                {
-                    attribute_name: 'id',
-                    key_type: 'HASH'
-                }
-            ],
-            attribute_definitions: [
-                {
-                    attribute_name: 'id',
-                    attribute_type: 'S'
-                }
-            ],
-            provisioned_throughput: {
-                read_capacity_units: 10,
-                write_capacity_units: 10
-            }
-        })
-
-        # wait here for the table to be ready
-        (1..10).each do |i|
-            sleep i
-            rsp = db.describe_table({
-                table_name: table
-            })
-            return true if rsp.to_h[:table][:table_status] == 'ACTIVE'
-        end
-    rescue => e
-        @logger.error "logstash-input-multirds ensure_lock_table exception\n #{e}"
-        return false
+      tables = db.list_tables({
+
+                              })
+      return true if tables.to_h[:table_names].to_a.include?(table)
+      # TODO: there is a potential race condition here where a table could come back in list_tables but not be in ACTIVE state we should check this better
+      db.create_table(
+        table_name: table,
+        key_schema: [
+          {
+            attribute_name: 'id',
+            key_type: 'HASH'
+          }
+        ],
+        attribute_definitions: [
+          {
+            attribute_name: 'id',
+            attribute_type: 'S'
+          }
+        ],
+        provisioned_throughput: {
+          read_capacity_units: 10,
+          write_capacity_units: 10
+        }
+      )
+
+      # wait here for the table to be ready
+      (1..10).each do |i|
+        sleep i
+        rsp = db.describe_table(
+          table_name: table
+        )
+        return true if rsp.to_h[:table][:table_status] == 'ACTIVE'
+      end
+    rescue StandardError => e
+      @logger.error "logstash-input-multirds ensure_lock_table exception\n #{e}"
+      return false
     end
-    return false
+    false
   end
-  def acquire_lock(db, table, id, lock_owner, expire_time: 10)
+  def acquire_lock(db, table, id, lock_owner, expire_time)
     begin
-        db.update_item({
-            key: {
-                id: id
-            },
-            table_name: table,
-            update_expression: "SET lock_owner = :lock_owner, expires = :expires",
-            expression_attribute_values: {
-                ':lock_owner' => lock_owner,
-                ':expires' =>  Time.now.utc.to_i + expire_time
-            },
-            return_values: "UPDATED_NEW",
-            condition_expression: "attribute_not_exists(lock_owner) OR lock_owner = :lock_owner OR expires < :expires"
-        })
-    rescue => e
+      db.update_item(
+        key: {
+          id: id
+        },
+        table_name: table,
+        update_expression: 'SET lock_owner = :lock_owner, expires = :expires',
+        expression_attribute_values: {
+          ':lock_owner' => lock_owner,
+          ':expires' => Time.now.utc.to_i + expire_time
+        },
+        return_values: 'UPDATED_NEW',
+        # condition_expression: 'attribute_not_exists(lock_owner) OR lock_owner = :lock_owner OR expires < :expires'
+        condition_expression: 'attribute_not_exists(lock_owner) OR expires < :expires'
+      )
+    # TODO: handle condition exception else throw error
+    rescue Aws::DynamoDB::Errors::ConditionalCheckFailedException
+      return false
+    rescue StandardError => e
       @logger.error "logstash-input-multirds acquire_lock exception\n #{e}"
+      false
     end
+    true
   end
+
   def get_logfile_list(rds, instance_pattern, logfile_pattern)
     log_files = []
     begin
       dbs = rds.describe_db_instances
       dbs.to_h[:db_instances].each do |db|
-          next unless db[:db_instance_identifier] =~ /#{instance_pattern}/
-          logs = rds.describe_db_log_files({
-              db_instance_identifier: db[:db_instance_identifier]
-          })
-
-          logs.to_h[:describe_db_log_files].each do |log|
-              next unless log[:log_file_name] =~ /#{logfile_pattern}/
-              log[:db_instance_identifier] = db[:db_instance_identifier]
-              log_files.push(log)
-          end
+        next unless db[:db_instance_identifier] =~ /#{instance_pattern}/
+        logs = rds.describe_db_log_files(
+          db_instance_identifier: db[:db_instance_identifier]
+        )
+
+        logs.to_h[:describe_db_log_files].each do |log|
+          next unless log[:log_file_name] =~ /#{logfile_pattern}/
+          log[:db_instance_identifier] = db[:db_instance_identifier]
+          log_files.push(log)
+        end
       end
-    rescue => e
+    rescue StandardError => e
       @logger.error "logstash-input-multirds get_logfile_list instance_pattern: #{instance_pattern} logfile_pattern:#{logfile_pattern} exception \n#{e}"
     end
     log_files
-end
-def get_logfile_record(db, id, tablename)
-  res = db.get_item({
-      key: {
+  end
+
+  def get_logfile_record(db, id, tablename)
+    out = {}
+    begin
+      res = db.get_item(
+        key: {
           id: id
-      },
-      table_name: tablename
-  })
-  extra_fields = {'marker' => '0:0'}
-  extra_fields.merge(res.item)
-end
-def set_logfile_record(db, id, tablename, key, value)
-  db.update_item({
-      key: {
+        },
+        table_name: tablename
+      )
+      extra_fields = { 'marker' => '0:0' }
+      out = extra_fields.merge(res.item)
+    rescue StandardError => e
+      @logger.error "logstash-input-multirds get_logfile_record  exception \n#{e}"
+    end
+    out
+  end
+
+  def set_logfile_record(db, id, tablename, key, value)
+    begin
+    out = db.update_item(
+        key: {
           id: id
-      },
-      table_name: tablename,
-      update_expression: "SET #{key} = :v",
-      expression_attribute_values: {
+        },
+        table_name: tablename,
+        update_expression: "SET #{key} = :v",
+        expression_attribute_values: {
           ':v' => value
-      },
-      return_values: "UPDATED_NEW"
-  
-  })
-end
+        },
+        return_values: 'UPDATED_NEW'
+      )
+    rescue StandardError => e
+      @logger.error "logstash-input-multirds set_logfile_record  exception \n#{e}"
+    end
+    out
+  end
 
   def register
-    @client_id = "#{Socket.gethostname}:#{java.util::UUID.randomUUID.to_s}" unless @client_id
-    @logger.info "Registering multi-rds input", :instance_name_pattern => @instance_name_pattern, :log_file_name_pattern => @log_file_name_pattern, :group_name => @group_name, :region => @region, :client_id => @client_id
+    @client_id ||= "#{Socket.gethostname}:#{java.util::UUID.randomUUID}"
+    @logger.info 'Registering multi-rds input', instance_name_pattern: @instance_name_pattern, log_file_name_pattern: @log_file_name_pattern, group_name: @group_name, region: @region, client_id: @client_id
 
     @db = Aws::DynamoDB::Client.new aws_options_hash
     @rds = Aws::RDS::Client.new aws_options_hash
@@ -139,8 +157,8 @@ end
   end
 
   def run(queue)
-    if !@ready
-      @logger.warn "multi-rds dynamodb lock table not ready, unable to proceed"
+    unless @ready
+      @logger.warn 'multi-rds dynamodb lock table not ready, unable to proceed'
       return false
     end
     @thread = Thread.current
@@ -149,67 +167,39 @@ end
 
       logs.each do |log|
         id = "#{log[:db_instance_identifier]}:#{log[:log_file_name]}"
-        lock = acquire_lock @db, @group_name, id, @client_id
+        lock = acquire_lock @db, @group_name, id, @client_id, (@polling_frequency - 1)
         next unless lock # we won't do anything with the data unless we get a lock on the file
-        
+
         rec = get_logfile_record @db, id, @group_name
         next unless rec['marker'].split(':')[1].to_i < log[:size].to_i # No new data in the log file so just continue
 
         # start reading log data at the marker
         more = true
         marker = rec[:marker]
-        while more do
-            rsp = @rds.download_db_log_file_portion(
-                db_instance_identifier: log[:db_instance_identifier],
-                log_file_name: log[:log_file_name],
-                marker: rec[:marker],
-            )
-            rsp[:log_file_data].lines.each do |line|
-              @codec.decode(line) do |event|
-                decorate event
-                event.set "rds_instance", log[:db_instance_identifier]
-                event.set "log_file", log[:log_file_name]
-                queue << event
-              end
+        while more
+          rsp = @rds.download_db_log_file_portion(
+            db_instance_identifier: log[:db_instance_identifier],
+            log_file_name: log[:log_file_name],
+            marker: rec[:marker]
+          )
+          rsp[:log_file_data].lines.each do |line|
+            @codec.decode(line) do |event|
+              decorate event
+              event.set 'rds_instance', log[:db_instance_identifier]
+              event.set 'log_file', log[:log_file_name]
+              queue << event
             end
-            more = rsp[:additional_data_pending]
-            marker = rsp[:marker] 
+          end
+          more = rsp[:additional_data_pending]
+          marker = rsp[:marker]
         end
         # set the marker back in the lock table
         set_logfile_record @db, id, @group_name, 'marker', marker
       end
     end
-
   end
 
   def stop
     Stud.stop! @thread
   end
-
-  def filename2datetime(name)
-    parts = name.match /(\d{4})-(\d{2})-(\d{2})-(\d{2})$/
-    Time.utc parts[1], parts[2], parts[3], parts[4]
-  end
-
-  private
-  module SinceDB
-    class File
-      def initialize(file)
-        @db = file
-      end
-
-      def read
-        if ::File.exists?(@db)
-          content = ::File.read(@db).chomp.strip
-          return content.empty? ? Time.new : Time.parse(content)
-        else
-          return Time.new("1999-01-01")
-        end
-      end
-
-      def write(time)
-        ::File.open(@db, 'w') { |file| file.write time.to_s }
-      end
-    end
-  end
 end

data/logstash-input-multirds.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirds'
-  s.version       = '0.0.2'
+  s.version       = '0.1.0'
   s.summary       = 'Ingest RDS log files to Logstash with competing consumers and multiple databases'
 
   s.authors       = ['Robert Labrie']
@@ -8,15 +8,15 @@ Gem::Specification.new do |s|
   s.homepage      = 'https://github.com/robertlabrie/logstash-input-multi-rds'
 
   s.require_paths = ['lib']
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir['lib/**/*', 'spec/**/*', 'vendor/**/*', '*.gemspec', '*.md', 'CONTRIBUTORS', 'Gemfile', 'LICENSE', 'NOTICE.TXT']
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'logstash_group' => 'input' }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
   s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
   s.add_runtime_dependency 'logstash-mixin-aws'
   s.add_runtime_dependency 'stud', '>= 0.0.22'
   s.add_development_dependency 'logstash-devutils', '>= 0.0.16'

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirds
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Robert Labrie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-21 00:00:00.000000000 Z
+date: 2018-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
+  name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: logstash-codec-plain
+  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: logstash-mixin-aws
   requirement: !ruby/object:Gem::Requirement