logstash-input-multirds 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e397a301602ab70c6241f3e12f97f0b748868ca3
-  data.tar.gz: fb55e7f3bccd472bbef7a8fc3a02b1277bf812d6
+  metadata.gz: 7dfd5f53cd59c0c7d8c89f79973a92bacc911b87
+  data.tar.gz: 184bb1f97be2b8d5896427d9ac4b5b92a585894b
 SHA512:
-  metadata.gz: 3ae26cdbf5cbab8f1a863a7cce692351b7775b1af471360c3df2e808c85a8b9c4828deaff09d4caca909e375094f9999533819b1f9427ecf990668e50ffb5f61
-  data.tar.gz: bc3d45b3f9fad096079233c7c766e66ff80060b1993dfb2f6f0361f11413404b083d6d6c2a9c496e95bc25debfc067671c783c2c8c02de92fcd6da614e4ea67a
+  metadata.gz: ef525b86cb6430c1e35776a1101c22d7f3355d36c95ca79f7dc3a839fd59223c9c025c76411bcfa591ca54a79dd19e18ece8107948de4099b10ad760273dedef
+  data.tar.gz: 6ef768857f2d767e864f42d8afadc513fb15d722dd565e891e9c9927893bcbca2c26213d5017be1b3ea79f3e696d7d8d8c43949c127fb560fedb56b7c734f1a9

data/README.md

@@ -1,11 +1,34 @@
 # Logstash Input Multi-RDS
 
-Forked from discourse/logstash-input-rds I needed competing consumer and multi-db support 
+Inputs RDS logs because Postgres doesn't support cloudwatch. Forked from discourse/logstash-input-rds I needed competing consumer and multi-db support. Uses DynamoDB for distributed locking and marker tracking. The plugin will create the table automatically if you give it permission, otherwise you can create it yourself where the table name is `group_name` and the primary key is a string called `id`.
+```
     input {
-      rds {
-        region => "us-west-2"
+      multirds {
+        region => "us-east-1"
         instance_name_pattern => ".*"
         log_file_name_pattern => ".*"
         group_name => "rds"
+
       }
     }
+```
+
+## Configuration
+
+* `region`: The AWS region for RDS. The AWS SDK reads this info from the usual places, so it's not required, but if you don't set it somewhere the plugin won't run
+  * **required**: false
+
+* `instance_name_pattern`: A regex pattern of RDS instances from which logs will be consumed
+  * **required**: false
+  * **default value**: `.*`
+
+* `log_file_name_pattern`: A regex pattern of RDS log files to consume
+  * **required**: false
+  * **default value**: `.*`
+
+* `group_name`: A unique identifier for all the instances of logstash which will be consuming this instance and log file pattern. Used for the lock table name.
+  * **required**: true
+
+* `client_id`: A unique identifier for a particular instance of logstash in the cluster
+  * **required**: false
+  * **default value**: `<hostname>:<uuid>`

data/lib/logstash/inputs/multirds.rb

@@ -3,10 +3,10 @@ require "logstash/inputs/base"
 require "logstash/namespace"
 require "stud/interval"
 require "aws-sdk"
-require "logstash/inputs/rds/patch"
+require "logstash/inputs/multirds/patch"
 require "logstash/plugin_mixins/aws_config"
 require "time"
-
+require "socket"
 Aws.eager_autoload!
 
 class LogStash::Inputs::Multirds < LogStash::Inputs::Base
@@ -16,55 +16,170 @@ class LogStash::Inputs::Multirds < LogStash::Inputs::Base
   milestone 1
   default :codec, "plain"
 
-  config :instance_name_pattern, :validate => :string, :required => true
-  config :log_file_name_pattern, :validate => :string, :required => true
+  config :instance_name_pattern, :validate => :string, :default => '.*'
+  config :log_file_name_pattern, :validate => :string, :default => '.*'
   config :polling_frequency, :validate => :number, :default => 600
   config :group_name, :validate => :string, :required => true
+  config :client_id, :validate => :string
+  
+  def ensure_lock_table(db, table)
+    begin
+        tables = db.list_tables({
+
+        })
+        return true if tables.to_h[:table_names].to_a.include?(table)
+        # TODO: there is a potential race condition here where a table could come back in list_tables but not be in ACTIVE state we should check this better
+        result = db.create_table({
+            table_name: table,
+            key_schema: [
+                {
+                    attribute_name: 'id',
+                    key_type: 'HASH'
+                }
+            ],
+            attribute_definitions: [
+                {
+                    attribute_name: 'id',
+                    attribute_type: 'S'
+                }
+            ],
+            provisioned_throughput: {
+                read_capacity_units: 10,
+                write_capacity_units: 10
+            }
+        })
+
+        # wait here for the table to be ready
+        (1..10).each do |i|
+            sleep i
+            rsp = db.describe_table({
+                table_name: table
+            })
+            return true if rsp.to_h[:table][:table_status] == 'ACTIVE'
+        end
+    rescue => e
+        @logger.error "logstash-input-multirds ensure_lock_table exception\n #{e}"
+        return false
+    end
+    return false
+  end
+  def acquire_lock(db, table, id, lock_owner, expire_time: 10)
+    begin
+        db.update_item({
+            key: {
+                id: id
+            },
+            table_name: table,
+            update_expression: "SET lock_owner = :lock_owner, expires = :expires",
+            expression_attribute_values: {
+                ':lock_owner' => lock_owner,
+                ':expires' =>  Time.now.utc.to_i + expire_time
+            },
+            return_values: "UPDATED_NEW",
+            condition_expression: "attribute_not_exists(lock_owner) OR lock_owner = :lock_owner OR expires < :expires"
+        })
+    rescue => e
+      @logger.error "logstash-input-multirds acquire_lock exception\n #{e}"
+    end
+  end
+  def get_logfile_list(rds, instance_pattern, logfile_pattern)
+    log_files = []
+    begin
+      dbs = rds.describe_db_instances
+      dbs.to_h[:db_instances].each do |db|
+          next unless db[:db_instance_identifier] =~ /#{instance_pattern}/
+          logs = rds.describe_db_log_files({
+              db_instance_identifier: db[:db_instance_identifier]
+          })
+
+          logs.to_h[:describe_db_log_files].each do |log|
+              next unless log[:log_file_name] =~ /#{logfile_pattern}/
+              log[:db_instance_identifier] = db[:db_instance_identifier]
+              log_files.push(log)
+          end
+      end
+    rescue => e
+      @logger.error "logstash-input-multirds get_logfile_list instance_pattern: #{instance_pattern} logfile_pattern:#{logfile_pattern} exception \n#{e}"
+    end
+    log_files
+end
+def get_logfile_record(db, id, tablename)
+  res = db.get_item({
+      key: {
+          id: id
+      },
+      table_name: tablename
+  })
+  extra_fields = {'marker' => '0:0'}
+  extra_fields.merge(res.item)
+end
+def set_logfile_record(db, id, tablename, key, value)
+  db.update_item({
+      key: {
+          id: id
+      },
+      table_name: tablename,
+      update_expression: "SET #{key} = :v",
+      expression_attribute_values: {
+          ':v' => value
+      },
+      return_values: "UPDATED_NEW"
   
+  })
+end
+
   def register
-    # @logger.info "Registering multi-RDS input", :region => @region, :instance => @instance_name, :log_file => @log_file_name
-    # @database = Aws::RDS::DBInstance.new @instance_name, aws_options_hash
-    # path = @sincedb_path || File.join(ENV["HOME"], ".sincedb_" + Digest::MD5.hexdigest("#{@instance_name}+#{@log_file_name}"))
-    # @sincedb = SinceDB::File.new path
-    @logger.info "Registering multi-rds input", :instance_name_pattern => @instance_name_pattern, :log_file_name_pattern => @log_file_name_pattern, :group_name = @group_name
-    @db = Aws::DynamoDB::Client.new
-    @rds = Aws::RDS::Client.new
-    # TODO: Auto-create dynamodb table here -- should that be a param? 
+    @client_id = "#{Socket.gethostname}:#{java.util::UUID.randomUUID.to_s}" unless @client_id
+    @logger.info "Registering multi-rds input", :instance_name_pattern => @instance_name_pattern, :log_file_name_pattern => @log_file_name_pattern, :group_name => @group_name, :region => @region, :client_id => @client_id
+
+    @db = Aws::DynamoDB::Client.new aws_options_hash
+    @rds = Aws::RDS::Client.new aws_options_hash
+
+    @ready = ensure_lock_table @db, @group_name
   end
 
   def run(queue)
+    if !@ready
+      @logger.warn "multi-rds dynamodb lock table not ready, unable to proceed"
+      return false
+    end
     @thread = Thread.current
     Stud.interval(@polling_frequency) do
-      @logger.debug "finding #{@log_file_name} for #{@instance_name} starting #{@sincedb.read} (#{@sincedb.read.to_i * 1000})"
-      begin
-        logfiles = @database.log_files({
-          filename_contains: @log_file_name,
-          file_last_written: @sincedb.read.to_i * 1000,
-        })
-        logfiles.each do |logfile|
-          @logger.debug "downloading #{logfile.name} for #{@instance_name}"
-          more = true
-          marker = "0"
-          while more do
-            response = logfile.download({marker: marker})
-            response[:log_file_data].lines.each do |line|
+      logs = get_logfile_list @rds, @instance_name_pattern, @log_file_name_pattern
+
+      logs.each do |log|
+        id = "#{log[:db_instance_identifier]}:#{log[:log_file_name]}"
+        lock = acquire_lock @db, @group_name, id, @client_id
+        next unless lock # we won't do anything with the data unless we get a lock on the file
+        
+        rec = get_logfile_record @db, id, @group_name
+        next unless rec['marker'].split(':')[1].to_i < log[:size].to_i # No new data in the log file so just continue
+
+        # start reading log data at the marker
+        more = true
+        marker = rec[:marker]
+        while more do
+            rsp = @rds.download_db_log_file_portion(
+                db_instance_identifier: log[:db_instance_identifier],
+                log_file_name: log[:log_file_name],
+                marker: rec[:marker],
+            )
+            rsp[:log_file_data].lines.each do |line|
               @codec.decode(line) do |event|
                 decorate event
-                event.set "rds_instance", @instance_name
-                event.set "log_file", @log_file_name
+                event.set "rds_instance", log[:db_instance_identifier]
+                event.set "log_file", log[:log_file_name]
                 queue << event
               end
             end
-            more = response[:additional_data_pending]
-            marker = response[:marker]
-          end
-          @sincedb.write (filename2datetime logfile.name)
+            more = rsp[:additional_data_pending]
+            marker = rsp[:marker] 
         end
-      rescue Aws::RDS::Errors::ServiceError
-        # the next iteration will resume at the same location
-        @logger.warn "caught AWS service error"
+        # set the marker back in the lock table
+        set_logfile_record @db, id, @group_name, 'marker', marker
       end
     end
+
   end
 
   def stop

data/logstash-input-multirds.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-multirds'
-  s.version       = '0.0.1'
+  s.version       = '0.0.2'
   s.summary       = 'Ingest RDS log files to Logstash with competing consumers and multiple databases'
 
   s.authors       = ['Robert Labrie']

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-multirds
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Robert Labrie