conrad 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b41619922ba6057aff0e0c1132b13ebd674e897f6edd141a7f4fc2ce6c8fad70
-  data.tar.gz: 381de5251326c319d50f1c4d6d2af93a1b00ffca594f5b3e423647c105cfb75a
+  metadata.gz: 5bea95c17248374dd5931d1de26433e47f3e47b46ef262ef89e86e8fe516040a
+  data.tar.gz: c1a1388a3ec0d4fcd8002a4b185f336950c92e037db2a831e8fa3cfee47627e9
 SHA512:
-  metadata.gz: f1d65e6a2141b159ee97077ecaa4cb9917edc060b5611736a17146e7f37daea7ce88984380d89ad6ddba30ef55ba3d1c7c81be717506bb1eafa1029e7c176f1a
-  data.tar.gz: 1455dbf8b1ac7b2a39c78c61fd000508d2cb222fa75c13aac7105f4a296a9667ccf2bc102bd9692583da303dacd8cee2aad03b28d4f68b431f503acb01bcf5e8
+  metadata.gz: 3cfc946e370b048f352936240516546ba5677d1e2bea493a19c8f537c93f358ad2198f33fb6ab02e6b9d590ad2169a8410fef0ce815a21b39d0b8ef5f7e0c32f
+  data.tar.gz: ba69a3a6d255e10700a7f6bc57672bfa3b06d51dbe2ffdf88ed6a24ebb0f536c16bf70489d9a892031b68daecc3d2c2f477d0706a42b3242b47f4262dadf9995

data/lib/conrad.rb

@@ -1,8 +1,25 @@
-Gem.find_files('conrad/processors/*.rb').each { |file| require file }
-Gem.find_files('conrad/formatters/*.rb').each { |file| require file }
-Gem.find_files('conrad/emitters/*.rb').each { |file| require file }
-Gem.find_files('conrad/*.rb').each { |file| require file }
+require 'conrad/version'
 
 # :nodoc:
 module Conrad
+  autoload :Collector,      'conrad/collector'
+  autoload :Errors,         'conrad/errors'
+  autoload :EmitterQueue,   'conrad/emitter_queue'
+  autoload :Emitters,       'conrad/emitters'
+  autoload :Formatters,     'conrad/formatters'
+  autoload :Processors,     'conrad/processors'
+  autoload :ProcessorStack, 'conrad/processor_stack'
+  autoload :Recorder,       'conrad/recorder'
+
+  class << self
+    # Boolean indicating if the events collected should be emitted in the
+    # background. Defaults to false.
+    def background_emit?
+      EmitterQueue.instance.background
+    end
+
+    def background_emit=(value)
+      EmitterQueue.instance.background = value
+    end
+  end
 end

data/lib/conrad/collector.rb

@@ -138,10 +138,12 @@ module Conrad
     #   emitted events, the error will be allowed to bubble up. This is to
     #   prevent the unexpected loss of events if a single one is malformed.
     def record_events
-      if emit_as_batch?
-        record_events_as_batch
-      else
-        record_individual_events
+      Array(emitter).each do |emitter|
+        if emit_as_batch?
+          record_events_as_batch(emitter, events.clone)
+        else
+          record_individual_events(emitter, events.clone)
+        end
       end
     ensure
       reset_state
@@ -173,16 +175,20 @@ module Conrad
       end
     end
 
-    def record_events_as_batch
-      emitter.call(events)
+    def record_events_as_batch(emitter, events)
+      EmitterQueue.instance.enqueue do
+        emitter.call(events)
+      end
     end
 
-    def record_individual_events
+    def record_individual_events(emitter, events)
       events.each do |event|
-        begin
-          emitter.call(event)
-        rescue StandardError => e
-          write_log(:error, e)
+        EmitterQueue.instance.enqueue do
+          begin
+            emitter.call(event)
+          rescue StandardError => e
+            write_log(:error, e)
+          end
         end
       end
     end

data/lib/conrad/emitter_queue.rb

@@ -0,0 +1,58 @@
+module Conrad
+  # Centralized event emission queue across threads
+  class EmitterQueue
+    include Singleton
+
+    # Boolean that determines whether events will be emitted inline or in a
+    #   background thread
+    attr_reader :background
+
+    # Logger object used for sending log events
+    attr_accessor :logger
+
+    def initialize
+      @thread = nil
+      @queue = Queue.new
+      @logger ||= Logger.new(STDOUT)
+    end
+
+    # bakground setter. Will start/stop the background thread
+    # @param value [Boolean] assigns whether events should be processed inline
+    #   or in a separate thread.
+    def background=(value)
+      @background = value
+      value ? start_thread : @thread = nil
+    end
+
+    # Enqueues a block
+    # @yield block to execute. Will either run inline or separately depending on
+    #   whether the queue is backgrounded
+    def enqueue
+      @queue.push -> { yield }
+
+      # if it's backgounded we can break out of here, as the background
+      #   queue will pick it up. otherwise, we need to explicitly process it
+      emit! unless @background
+    end
+
+    private
+
+    def start_thread
+      @thread ||= Thread.new do
+        Thread.current.abort_on_exception = true
+        loop do
+          emit!
+          break unless @background
+        end
+      end
+    rescue e
+      logger.error(e)
+      @thread = nil
+      start_thread
+    end
+
+    def emit!
+      @queue.pop.call until @queue.empty?
+    end
+  end
+end

data/lib/conrad/emitters.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Conrad
+  # :nodoc:
+  module Emitters
+    autoload :AmazonBase, 'conrad/emitters/amazon_base'
+    autoload :Kinesis,    'conrad/emitters/kinesis'
+    autoload :Sqs,        'conrad/emitters/sqs'
+    autoload :Stdout,     'conrad/emitters/stdout'
+  end
+end

data/lib/conrad/emitters/amazon_base.rb

@@ -0,0 +1,59 @@
+require 'active_model'
+
+module Conrad
+  module Emitters
+    # Base class for AWS-based emitters
+    class AmazonBase
+      include ::ActiveModel::Model
+
+      # @return [String, nil] the configured region
+      attr_accessor :region
+
+      # @deprecated Will be removed in 3.0.0, no migration path
+      # @return [String, nil] the configured AWS Access key ID
+      attr_accessor :access_key_id
+
+      # @deprecated Will be removed in 3.0.0, no migration path
+      # @return [String, nil] the configured AWS secret access key
+      attr_accessor :secret_access_key
+
+      # @return [Aws::SQS::Client] the created client
+      attr_accessor :client
+
+      # @param queue_url [String] the queue to send messages to
+      # @param region [String] region the queue lives in
+      # @param access_key_id [String] AWS Acesss Key ID
+      # @param secret_access_key [String] AWS Secret Access Key
+      #
+      # @raise [InvalidAwsCredentials] if access_key_id or secret_access_key are
+      #   not provided AND the running environment does not have valid AWS
+      #   credentials
+      # @raise [Aws::Errors::MissingRegionError] if region is not provided and
+      #   also not set via an allowed AWS environment variable
+      def initialize(args = {})
+        super
+        create_client(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
+      end
+
+      private
+
+      def create_client(region:, access_key_id:, secret_access_key:)
+        if secret_access_key.nil? || access_key_id.nil?
+          validate_implicit_credentials
+
+          @client = self.class.client_class.new({ region: region }.compact)
+        else
+          @client = self.class.client_class.new({
+            region: region,
+            access_key_id: access_key_id,
+            secret_access_key: secret_access_key
+          }.compact)
+        end
+      end
+
+      def validate_implicit_credentials
+        raise Conrad::InvalidAwsCredentials unless Aws::CredentialProviderChain.new.resolve&.set?
+      end
+    end
+  end
+end

data/lib/conrad/emitters/kinesis.rb

@@ -0,0 +1,34 @@
+require 'conrad/errors'
+
+module Conrad
+  # A module containing all of conrad's built in event emitters for outputting
+  # events
+  module Emitters
+    # Basic emitter for sending events to AWS's kinesis event stream. The emitter will
+    # attempt to use values configured in the running environment according to
+    # the AWS SDK documentation (such as from ~/.aws/credentials).
+    class Kinesis < AmazonBase
+      # @return [String] the configured kinesis stream name
+      attr_accessor :stream_name
+
+      # Sends an event up to Kinesis
+      #
+      # @param event [String] the event to be sent as a Kinesis message body
+      def call(event)
+        client.put_record(
+          stream_name: stream_name,
+          data: event,
+          # There's a 256 character limit on the partition key, and it's hashed down into a value used to
+          # pick the shard to put the data on
+          partition_key: event.first(255)
+        )
+      end
+
+      class << self
+        def client_class
+          Aws::Kinesis::Client
+        end
+      end
+    end
+  end
+end

data/lib/conrad/emitters/sqs.rb

@@ -8,51 +8,9 @@ module Conrad
     # is given, the given credentials will be used. Otherwise, the emitter will
     # attempt to use values configured in the running environment according to
     # the AWS SDK documentation (such as from ~/.aws/credentials).
-    class Sqs
-      # Error for responding with issues around SQS credential creation
-      class InvalidAwsCredentials < ::Conrad::Error
-        # :nodoc:
-        def to_s
-          'Must provide secret_access_key and access_key_id OR rely ' \
-          'on configured values in the running environment.'
-        end
-      end
-
+    class Sqs < AmazonBase
       # @return [String] the configured SQS queue URL
-      attr_reader :queue_url
-
-      # @return [String, nil] the configured region
-      attr_reader :region
-
-      # @deprecated Will be removed in 3.0.0, no migration path
-      # @return [String, nil] the configured AWS Access key ID
-      attr_reader :access_key_id
-
-      # @deprecated Will be removed in 3.0.0, no migration path
-      # @return [String, nil] the configured AWS secret access key
-      attr_reader :secret_access_key
-
-      # @return [Aws::SQS::Client] the created client
-      attr_reader :client
-
-      # @param queue_url [String] the queue to send messages to
-      # @param region [String] region the queue lives in
-      # @param access_key_id [String] AWS Acesss Key ID
-      # @param secret_access_key [String] AWS Secret Access Key
-      #
-      # @raise [InvalidAwsCredentials] if access_key_id or secret_access_key are
-      #   not provided AND the running environment does not have valid AWS
-      #   credentials
-      # @raise [Aws::Errors::MissingRegionError] if region is not provided and
-      #   also not set via an allowed AWS environment variable
-      def initialize(queue_url:, region: nil, access_key_id: nil, secret_access_key: nil)
-        @queue_url = queue_url
-        @region = region
-        @access_key_id = access_key_id
-        @secret_access_key = secret_access_key
-
-        create_client(region: region, access_key_id: access_key_id, secret_access_key: secret_access_key)
-      end
+      attr_accessor :queue_url
 
       # Sends an event up to SQS
       #
@@ -61,25 +19,11 @@ module Conrad
         client.send_message(queue_url: queue_url, message_body: event)
       end
 
-      private
-
-      def create_client(region:, access_key_id:, secret_access_key:)
-        if secret_access_key.nil? || access_key_id.nil?
-          validate_implicit_credentials
-
-          @client = Aws::SQS::Client.new({ region: region }.compact)
-        else
-          @client = Aws::SQS::Client.new({
-            region: region,
-            access_key_id: access_key_id,
-            secret_access_key: secret_access_key
-          }.compact)
+      class << self
+        def client_class
+          Aws::SQS::Client
         end
       end
-
-      def validate_implicit_credentials
-        raise InvalidAwsCredentials unless Aws::CredentialProviderChain.new.resolve.set?
-      end
     end
   end
 end

data/lib/conrad/errors.rb

@@ -9,6 +9,15 @@ module Conrad
     end
   end
 
+  # Error for responding with issues around kinesis credential creation
+  class InvalidAwsCredentials < Error
+    # :nodoc:
+    def to_s
+      'Must provide secret_access_key and access_key_id OR rely ' \
+      'on configured values in the running environment.'
+    end
+  end
+
   # Error raised when the value of an event attribute is not of one of the
   # allowed types
   class ForbiddenValue < Error

data/lib/conrad/formatters.rb

@@ -0,0 +1,6 @@
+module Conrad
+  # :nodoc:
+  module Formatters
+    autoload :JSON, 'conrad/formatters/json'
+  end
+end

data/lib/conrad/processors.rb

@@ -1 +1,8 @@
-Dir['conrad/processors/*.rb'].each { |file| require file }
+module Conrad
+  # :nodoc:
+  module Processors
+    autoload :AddTimestamp, 'conrad/processors/add_timestamp'
+    autoload :AddUUID,      'conrad/processors/add_uuid'
+    autoload :Envelope,     'conrad/processors/envelope'
+  end
+end

data/lib/conrad/version.rb

@@ -1,5 +1,5 @@
 # :nodoc:
 module Conrad
   # :nodoc:
-  VERSION = '2.3.1'.freeze
+  VERSION = '2.4.0'.freeze
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: conrad
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.4.0
 platform: ruby
 authors:
 - Jonathon Anderson
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-28 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -89,9 +103,14 @@ extra_rdoc_files: []
 files:
 - lib/conrad.rb
 - lib/conrad/collector.rb
+- lib/conrad/emitter_queue.rb
+- lib/conrad/emitters.rb
+- lib/conrad/emitters/amazon_base.rb
+- lib/conrad/emitters/kinesis.rb
 - lib/conrad/emitters/sqs.rb
 - lib/conrad/emitters/stdout.rb
 - lib/conrad/errors.rb
+- lib/conrad/formatters.rb
 - lib/conrad/formatters/json.rb
 - lib/conrad/processor_stack.rb
 - lib/conrad/processors.rb
@@ -120,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.7.3
 signing_key: 
 specification_version: 4
 summary: Tool for auditing events.