event_tracer 0.2.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 77058f3fa969027f3d51399425d8885c60037ae6250db1cfa686fbafb8524467
-  data.tar.gz: 40b98f105722939c88bb364a04bfbd708c9fcfc2c76815d41d5b2318994aa9bb
+  metadata.gz: '002520584662d6b31ce2f05310a70c4fb8af37c3c865f6bdf2b8ebc44bf16654'
+  data.tar.gz: daa3fc169b3daa2771adb7d832872c055b723dbe25d28fff4ba24c618f2b54a2
 SHA512:
-  metadata.gz: 31fb42ad32f68d2efd4b78dbfb9d55f16563bc897eaa4c4602c5855eb092bf0f03c4308c22a91ed011663c095548029155a7a795de638405588bb162a93d723b
-  data.tar.gz: 58c9ec65b708b9b44fb141c9da778fbc6d56fea371a8edc9e0aaf9ef4d4a2b89855e0a88afbafd0a61312c2b8023bd7c13325bc0f7c86be13ee7d87e534dc6ad
+  metadata.gz: '038e91c1d03b17070b97c3c170d73992d4050b9cee39f3529fc712a2cfda324d8afdfb10013f6a6838176966f94d6dd17b5f4851a62af6bdc0e6c8ea2b2e95b1'
+  data.tar.gz: fdeb28420163369840b82b0a56e6bd83a638fe4c5e7069f280c6807d8f9ad00b9044c642263fc89391628c110af8e527596c2e86c2b759a50691c3d13a7390c5

data/lib/event_tracer/appsignal_logger.rb

@@ -2,62 +2,62 @@ require_relative '../event_tracer'
 require_relative './basic_decorator'
 
 # NOTES
-# Appsignal interface to send our usual actions 
+# Appsignal interface to send our usual actions
 # BasicDecorator adds a transparent interface on top of the appsignal interface
 #
-# Usage: EventTracer.register :appsignal, EventTracer::AppsignalLogger.new(Appsignal)
-#        appsignal_logger.info appsignal: { increment_counter: { counter_1: 1, counter_2: 2 }, set_gauge: { gauge_1: 1 } }
-#        appsignal_logger.info appsignal: { set_gauge: { gauge_1: { value: 1, tags: { region: 'eu' } } } }
+# Usage: EventTracer.register :appsignal,
+#          EventTracer::AppsignalLogger.new(Appsignal, allowed_tags: ['tag_1', 'tag_2'])
+#
+#        appsignal_logger.info metrics: [:counter_1, :counter_2]
+#        appsignal_logger.info metrics: { counter_1: { type: :counter, value: 1 }, gauce_2: { type: :gauce, value: 10 } }
 module EventTracer
   class AppsignalLogger < BasicDecorator
 
-    SUPPORTED_METRICS ||= %i(increment_counter add_distribution_value set_gauge)
+    SUPPORTED_METRIC_TYPES = {
+      counter: :increment_counter,
+      distribution: :add_distribution_value,
+      gauge: :set_gauge
+    }
+    DEFAULT_METRIC_TYPE = :increment_counter
+    DEFAULT_COUNTER = 1
+
+    def initialize(decoratee, allowed_tags: [])
+      super(decoratee)
+      @allowed_tags = allowed_tags
+    end
 
     LOG_TYPES.each do |log_type|
       define_method log_type do |**args|
-        return LogResult.new(false, "Invalid appsignal config") unless args[:appsignal] && args[:appsignal].is_a?(Hash)
+        metrics = args[:metrics]
+
+        return fail_result('Invalid appsignal config') unless valid_args?(metrics)
+        return success_result if metrics.empty?
 
-        applied_metrics(args[:appsignal]).each do |metric|
-          metric_args = args[:appsignal][metric]
-          return LogResult.new(false, "Appsignal metric #{metric} invalid") unless metric_args && metric_args.is_a?(Hash) 
+        tags = args.slice(*allowed_tags)
 
-          send_metric metric, metric_args
-        rescue InvalidTagError => e
-          return LogResult.new(false, e.message)
+        case metrics
+        when Array
+          metrics.each do |metric|
+            appsignal.public_send(DEFAULT_METRIC_TYPE, metric, DEFAULT_COUNTER, tags)
+          end
+        when Hash
+          metrics.each do |metric_name, metric_payload|
+            metric_type = SUPPORTED_METRIC_TYPES[metric_payload.fetch(:type).to_sym]
+            appsignal.public_send(metric_type, metric_name, metric_payload.fetch(:value), tags) if metric_type
+          end
         end
 
-        LogResult.new(true)
+        success_result
       end
     end
 
     private
 
-      attr_reader :appsignal, :decoratee
+      attr_reader :decoratee, :allowed_tags
       alias_method :appsignal, :decoratee
 
-      def applied_metrics(appsignal_args)
-        appsignal_args.keys.select { |metric| SUPPORTED_METRICS.include?(metric) }
-      end
-
-      def send_metric(metric, payload)
-        payload.each do |increment, attribute|
-          if attribute.is_a?(Hash)
-            begin
-              appsignal.send(
-                metric,
-                increment,
-                attribute.fetch(:value),
-                attribute.fetch(:tags)
-              )
-            rescue KeyError
-              raise InvalidTagError, "Appsignal payload { #{increment}: #{attribute} } invalid"
-            end
-          else
-            appsignal.send(metric, increment, attribute)
-          end
-        end
+      def valid_args?(metrics)
+        metrics && (metrics.is_a?(Hash) || metrics.is_a?(Array))
       end
   end
-
-  class InvalidTagError < StandardError; end
 end

data/lib/event_tracer/base_logger.rb

@@ -7,8 +7,8 @@ module EventTracer
   class BaseLogger < BasicDecorator
 
     LOG_TYPES.each do |log_type|
-      define_method log_type do |*args|
-        send_message(log_type, *args)
+      define_method log_type do |**args|
+        send_message(log_type, **args)
         LogResult.new(true)
       end
     end
@@ -20,6 +20,7 @@ module EventTracer
 
       # EventTracer ensures action & message is always populated
       def send_message(log_method, action:, message:, **args)
+        args.delete(:metrics)
         data = args.merge(action: action, message: message)
         logger.public_send(log_method, data)
       end

data/lib/event_tracer/basic_decorator.rb

@@ -1,3 +1,5 @@
+require 'delegate'
+
 module EventTracer
   class BasicDecorator < Delegator
 
@@ -14,5 +16,13 @@ module EventTracer
       @delegate_sd_obj = obj
     end
 
+    def success_result
+      LogResult.new(true)
+    end
+
+    def fail_result(message)
+      LogResult.new(false, message)
+    end
+
   end
-end
+end

data/lib/event_tracer/buffer.rb

@@ -0,0 +1,65 @@
+require 'concurrent'
+
+module EventTracer
+  # This is an implementation of buffer storage. We use Concurrent::Array underneath
+  # to ensure thread-safe behavior. Data is stored until certain size / interval
+  # before flushing.
+  #
+  # Caveats: We should only store non-important data like logs in this buffer
+  # because if a process is killed, the data in this buffer is lost.
+  class Buffer
+    # Buffer can store maximum 10 items.
+    # Bigger size requires more memory to store, so choose a reasonable number
+    DEFAULT_BUFFER_SIZE = 10
+    # An item can live in buffer for at least 10s between each `Buffer#add` if the buffer is not full
+    # If there are larger interval between the calls, it can live longer.
+    DEFAULT_FLUSH_INTERVAL = 10
+
+    def initialize(
+      buffer_size: DEFAULT_BUFFER_SIZE,
+      flush_interval: DEFAULT_FLUSH_INTERVAL
+    )
+      @buffer_size = buffer_size
+      @flush_interval = flush_interval
+      @buffer = Concurrent::Array.new
+    end
+
+    # Add an item to buffer
+    #
+    # @param item: data to be added to buffer
+    # @return true if the item can be added, otherwise false
+    def add(item)
+      if add_item?
+        buffer.push({ item: item, created_at: Time.now })
+        true
+      else
+        false
+      end
+    end
+
+    # Remove all existing items from buffer
+    #
+    # @return all items in buffer
+    def flush
+      data = []
+
+      data << buffer.shift[:item] until buffer.empty?
+
+      data
+    end
+
+    # This method is only used to facilitate testing
+    def size
+      buffer.size
+    end
+
+    private
+
+      attr_reader :buffer_size, :flush_interval, :buffer
+
+      def add_item?
+        buffer.size < buffer_size &&
+          (buffer.empty? || buffer.first[:created_at] > Time.now - flush_interval)
+      end
+  end
+end

data/lib/event_tracer/config.rb

@@ -0,0 +1,10 @@
+require 'dry-configurable'
+
+module EventTracer
+  class Config
+    extend Dry::Configurable
+
+    setting :app_name, default: 'app_name'
+    setting :dynamo_db_table_name, default: 'logs'
+  end
+end

data/lib/event_tracer/datadog_logger.rb

@@ -4,72 +4,68 @@ require_relative './basic_decorator'
 # Datadog interface to send our usual actions
 # BasicDecorator adds a transparent interface on top of the datadog interface
 #
-# Usage: EventTracer.register :datadog, EventTracer::DataDogLogger.new(DataDog)
-#        data_dog_logger.info datadog: { count: { counter_1: 1, counter_2: 2 }, set: { gauge_1: 1 } }
-#        data_dog_logger.info datadog: { count: { counter_1: { value: 1, tags: ['tag1, tag2']} } }
+# Usage: EventTracer.register :datadog,
+#          EventTracer::DataDogLogger.new(DataDog, allowed_tags: ['tag_1', 'tag_2'])
+#
+#        data_dog_logger.info metrics: [:counter_1, :counter_2]
+#        data_dog_logger.info metrics: { counter_1: { type: :counter, value: 1}, gauce_2: { type: :gauce, value: 10 } }
 
 module EventTracer
   class DatadogLogger < BasicDecorator
 
-    class InvalidTagError < StandardError; end
+    SUPPORTED_METRIC_TYPES = {
+      counter: :count,
+      distribution: :distribution,
+      gauge: :gauge,
+      set: :set,
+      histogram: :histogram
+    }
+    DEFAULT_METRIC_TYPE = :count
+    DEFAULT_COUNTER = 1
 
-    SUPPORTED_METRICS ||= %i[count set distribution gauge histogram].freeze
+    def initialize(decoratee, allowed_tags: [])
+      super(decoratee)
+      @allowed_tags = allowed_tags
+    end
 
     LOG_TYPES.each do |log_type|
       define_method log_type do |**args|
-        return LogResult.new(false, 'Invalid datadog config') unless args[:datadog]&.is_a?(Hash)
+        metrics = args[:metrics]
+
+        return fail_result('Invalid Datadog config') unless valid_args?(metrics)
+        return success_result if metrics.empty?
 
-        applied_metrics(args[:datadog]).each do |metric|
-          metric_args = args[:datadog][metric]
-          return LogResult.new(false, "Datadog metric #{metric} invalid") unless metric_args.is_a?(Hash)
+        tags = build_tags(args)
 
-          send_metric metric, metric_args
+        case metrics
+        when Array
+          metrics.each do |metric|
+            datadog.public_send(DEFAULT_METRIC_TYPE, metric, DEFAULT_COUNTER, tags: tags)
+          end
+        when Hash
+          metrics.each do |metric_name, metric_payload|
+            metric_type = SUPPORTED_METRIC_TYPES[metric_payload.fetch(:type).to_sym]
+            datadog.public_send(metric_type, metric_name, metric_payload.fetch(:value), tags: tags) if metric_type
+          end
         end
 
-        LogResult.new(true)
+        success_result
       end
     end
 
     private
 
-    attr_reader :datadog, :decoratee
+    attr_reader :decoratee, :allowed_tags
     alias_method :datadog, :decoratee
 
-    def applied_metrics(datadog_args)
-      datadog_args.keys.select { |metric| SUPPORTED_METRICS.include?(metric) }
+    def valid_args?(metrics)
+      metrics && (metrics.is_a?(Hash) || metrics.is_a?(Array))
     end
 
-    def send_metric(metric, payload)
-      payload.each do |increment, attribute|
-        if attribute.is_a?(Hash)
-          begin
-            datadog.public_send(
-              metric,
-              increment,
-              attribute.fetch(:value),
-              build_options(attribute[:tags])
-            )
-          rescue KeyError
-            raise InvalidTagError, "Datadog payload { #{increment}: #{attribute} } invalid"
-          end
-        else
-          datadog.public_send(metric, increment, attribute)
-        end
+    def build_tags(args)
+      args.slice(*allowed_tags).map do |tag, value|
+        "#{tag}:#{value}"
       end
     end
-
-    def build_options(tags)
-      return {} unless tags
-
-      formattted_tags =
-        if tags.is_a?(Array)
-          tags
-        else
-          tags.inject([]) do |acc, (tag, value)|
-            acc << "#{tag}:#{value}"
-          end
-        end
-      { tags: formattted_tags }
-    end
   end
 end

data/lib/event_tracer/dynamo_db_client.rb

@@ -0,0 +1,7 @@
+module EventTracer
+  class DynamoDBClient
+    def self.call
+      Aws::DynamoDB::Client.new
+    end
+  end
+end

data/lib/event_tracer/dynamo_db_default_processor.rb

@@ -0,0 +1,13 @@
+module EventTracer
+  class DynamoDBDefaultProcessor
+    def call(log_type, action:, message:, args:)
+      args.merge(
+        timestamp: Time.now.utc.iso8601(6),
+        action: action,
+        message: message,
+        log_type: log_type,
+        app: EventTracer::Config.config.app_name
+      )
+    end
+  end
+end

data/lib/event_tracer/dynamo_db_log_worker.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require_relative 'dynamo_db_client'
+
+begin
+  require 'sidekiq'
+  require 'aws-sdk-dynamodb'
+rescue LoadError => e
+  puts "Please add the missing gem into your app Gemfile: #{e.message}"
+  raise
+end
+
+module EventTracer
+  class DynamoDBLogWorker
+    include ::Sidekiq::Worker
+
+    sidekiq_options retry: 1, queue: 'low'
+
+    # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+    MAX_DYNAMO_DB_ITEM_PER_REQUEST = 25
+
+    def perform(items)
+      wrap(items).each_slice(MAX_DYNAMO_DB_ITEM_PER_REQUEST) do |batch|
+        data = batch.map do |item|
+          { put_request: { item: clean_empty_values(item) } }
+        end
+
+        EventTracer::DynamoDBClient.call.batch_write_item(
+          request_items: { EventTracer::Config.config.dynamo_db_table_name => data }
+        )
+
+      rescue Aws::DynamoDB::Errors::ServiceError => e
+        EventTracer.error(
+          loggers: %i(base),
+          action: 'DynamoDBLogWorker',
+          app: EventTracer::Config.config.app_name,
+          error: e.class.name,
+          message: e.message
+        )
+      end
+    end
+
+    private
+
+      def wrap(items)
+        # NOTE: This allows us to handle either buffered or unbuffered payloads
+        if items.is_a?(Hash)
+          [items]
+        else
+          Array(items)
+        end
+      end
+
+      # dynamo can't serialise empty strings/ non-zero numerics
+      def clean_empty_values(data)
+        data.delete_if do |_key, value|
+          case value
+          when Hash
+            clean_empty_values(value)
+            false
+          when String then value.empty?
+          else false
+          end
+        end
+      end
+
+  end
+end

data/lib/event_tracer/dynamo_db_logger.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'dynamo_db_client'
+
+module EventTracer
+  class DynamoDBLogger
+    def initialize(buffer: Buffer.new(buffer_size: 0), log_processor: EventTracer::DynamoDBDefaultProcessor.new)
+      @buffer = buffer
+      @log_processor = log_processor
+    end
+
+    EventTracer::LOG_TYPES.each do |log_type|
+      define_method log_type do |**args|
+        save_message log_type, **args
+      end
+    end
+
+    private
+
+      attr_reader :buffer, :log_processor
+
+      def save_message(log_type, action:, message:, **args)
+        payload = log_processor.call(log_type, action: action, message: message, args: args)
+
+        unless buffer.add(payload)
+          all_payloads = buffer.flush + [payload]
+          DynamoDBLogWorker.perform_async(all_payloads)
+        end
+
+        LogResult.new(true)
+      end
+
+  end
+end

data/lib/event_tracer/version.rb

@@ -1,3 +1,3 @@
 module EventTracer
-  VERSION = '0.2.1'.freeze
+  VERSION = '0.3.2'.freeze
 end

data/lib/event_tracer.rb

@@ -31,14 +31,10 @@ module EventTracer
       result = Result.new
 
       loggers.each do |code, logger|
-        begin
-          if args[:action] && args[:message]
-            result.record code, logger.send(log_type, filtered_log_arguments(code, args))
-          else
-            result.record code, LogResult.new(false, 'Fields action & message need to be populated')
-          end
-        rescue Exception => e
-          result.record code, LogResult.new(false, e.message)
+        if args[:action] && args[:message]
+          result.record code, logger.send(log_type, **args)
+        else
+          result.record code, LogResult.new(false, 'Fields action & message need to be populated')
         end
       end
 
@@ -55,14 +51,6 @@ module EventTracer
       @loggers.select { |code, _logger| selected_codes.include?(code) }
     end
 
-    def self.filtered_log_arguments(logger_code, args)
-      blacklisted_logger_keys = registered_logger_codes - [logger_code]
-
-      args.reject do |key, _value|
-        blacklisted_logger_keys.include?(key)
-      end
-    end
-
     def self.registered_logger_codes
       @loggers.keys
     end

metadata

@@ -1,27 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: event_tracer
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.2
 platform: ruby
 authors:
 - melvrickgoh
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-11-15 00:00:00.000000000 Z
+date: 2021-10-11 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-configurable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.1.4
 - !ruby/object:Gem::Dependency
@@ -53,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 description: 'Thin wrapper for formatted logging/ metric services to be used as a
-  single service. External service(s) supported: Appsignal'
+  single service. External service(s) supported: Appsignal, Datadog, DynamoDB'
 email:
 - melvrickgoh@hotmail.com
 executables: []
@@ -64,7 +92,13 @@ files:
 - lib/event_tracer/appsignal_logger.rb
 - lib/event_tracer/base_logger.rb
 - lib/event_tracer/basic_decorator.rb
+- lib/event_tracer/buffer.rb
+- lib/event_tracer/config.rb
 - lib/event_tracer/datadog_logger.rb
+- lib/event_tracer/dynamo_db_client.rb
+- lib/event_tracer/dynamo_db_default_processor.rb
+- lib/event_tracer/dynamo_db_log_worker.rb
+- lib/event_tracer/dynamo_db_logger.rb
 - lib/event_tracer/log_result.rb
 - lib/event_tracer/version.rb
 homepage: https://github.com/melvrickgoh/event_tracer
@@ -87,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
+rubygems_version: 3.2.28
 signing_key:
 specification_version: 4
 summary: Thin wrapper for formatted logging/ metric services to be used as a single