event_tracer 0.3.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5207c9165b5c41811f3128fdf9d22c1dd21f805131aab1d2dfa2861bc55a43b
-  data.tar.gz: 3cda4e4b5dcb3c994797e9cae5ef3d31253e49d09a5d7527b49f107fc430e1df
+  metadata.gz: c33ea83663eda9d96f8055d957ab688d6be6f957023ea689b5f119abedfb7ad8
+  data.tar.gz: f6f328085fdc3de07dafa12e27bac3752b18249bcd5bbd3cd81046222c853f19
 SHA512:
-  metadata.gz: 6333e12e55d047a9d4b6f5a6cb8af0781218a5be884b06a1b96fa3e994a73901f72d14e47269929df96284c0a8bb5ffc8f7aa6677413d2952a87b1b54b9f81c4
-  data.tar.gz: 8a27bbcf11291c1ca89c39944b784fb44661b00496f10412da4efbe933c7a014bac7a7fa186d55580e0fd2e766b1fcbe18565c13475091d1a01e6c1d0369c653
+  metadata.gz: 49bf70ea7a94f1a88410b8b014c77763fd4f9f581503fb48500c07e02000db3325c3b113baa1df5f733b1326e02e28441137512972f8e1b8c07a0917f662c59c
+  data.tar.gz: 70a852a9f2de92aa90ce63381ac7b63252d0f889b9f0bf34eb51d4d43e4e9740b5532029571992150ebb50dae035fb1a9a5b12383c3e48eaeb25c15045cffcb5

data/lib/event_tracer/buffer.rb

@@ -0,0 +1,65 @@
+require 'concurrent'
+
+module EventTracer
+  # This is an implementation of buffer storage. We use Concurrent::Array underneath
+  # to ensure thread-safe behavior. Data is stored until certain size / interval
+  # before flushing.
+  #
+  # Caveats: We should only store non-important data like logs in this buffer
+  # because if a process is killed, the data in this buffer is lost.
+  class Buffer
+    # Buffer can store maximum 10 items.
+    # Bigger size requires more memory to store, so choose a reasonable number
+    DEFAULT_BUFFER_SIZE = 10
+    # An item can live in buffer for at least 10s between each `Buffer#add` if the buffer is not full
+    # If there are larger interval between the calls, it can live longer.
+    DEFAULT_FLUSH_INTERVAL = 10
+
+    def initialize(
+      buffer_size: DEFAULT_BUFFER_SIZE,
+      flush_interval: DEFAULT_FLUSH_INTERVAL
+    )
+      @buffer_size = buffer_size
+      @flush_interval = flush_interval
+      @buffer = Concurrent::Array.new
+    end
+
+    # Add an item to buffer
+    #
+    # @param item: data to be added to buffer
+    # @return true if the item can be added, otherwise false
+    def add(item)
+      if add_item?
+        buffer.push({ item: item, created_at: Time.now })
+        true
+      else
+        false
+      end
+    end
+
+    # Remove all existing items from buffer
+    #
+    # @return all items in buffer
+    def flush
+      data = []
+
+      data << buffer.shift[:item] until buffer.empty?
+
+      data
+    end
+
+    # This method is only used to facilitate testing
+    def size
+      buffer.size
+    end
+
+    private
+
+      attr_reader :buffer_size, :flush_interval, :buffer
+
+      def add_item?
+        buffer.size < buffer_size &&
+          (buffer.empty? || buffer.first[:created_at] > Time.now - flush_interval)
+      end
+  end
+end

data/lib/event_tracer/buffered_logger.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module EventTracer
+  class BufferedLogger
+    def initialize(log_processor:, worker:, buffer: Buffer.new(buffer_size: 0))
+      @buffer = buffer
+      @worker = worker
+      @log_processor = log_processor
+    end
+
+    EventTracer::LOG_TYPES.each do |log_type|
+      define_method log_type do |**args|
+        save_message log_type, **args
+      end
+    end
+
+    private
+
+    attr_reader :buffer, :log_processor, :worker
+
+    def save_message(log_type, action:, message:, **args)
+      payload = log_processor.call(log_type, action: action, message: message, args: args)
+
+      unless buffer.add(payload)
+        all_payloads = buffer.flush + [payload]
+        execute_payload(all_payloads)
+      end
+
+      LogResult.new(true)
+    end
+
+    def execute_payload(payloads)
+      worker.perform_async(payloads)
+    rescue JSON::GeneratorError => e
+      filtered_payloads = filter_invalid_data(payloads)
+
+      EventTracer.warn(
+        loggers: %i(base),
+        action: self.class.name,
+        app: EventTracer::Config.config.app_name,
+        error: e.class.name,
+        message: e.message,
+        payload: payloads - filtered_payloads
+      )
+
+      worker.perform_async(filtered_payloads) if filtered_payloads.any?
+    end
+
+    def filter_invalid_data(payloads)
+      payloads.select { |payload| payload.to_json rescue false }
+    end
+  end
+end

data/lib/event_tracer/config.rb

@@ -0,0 +1,14 @@
+require 'dry-configurable'
+
+module EventTracer
+  class Config
+    extend Dry::Configurable
+
+    setting :app_name, default: 'app_name'
+
+    # TODO: switch to namespace in v1.0
+    setting :dynamo_db_table_name, default: 'logs'
+    setting :dynamo_db_client
+    setting :dynamo_db_queue_name, default: 'low'
+  end
+end

data/lib/event_tracer/dynamo_db/client.rb

@@ -0,0 +1,14 @@
+module EventTracer
+  module DynamoDB
+    class Client
+      class << self
+        extend Gem::Deprecate
+
+        def call
+          Aws::DynamoDB::Client.new
+        end
+        deprecate :call, 'EventTracer::Config.config.dynamo_db_client', 2021, 12
+      end
+    end
+  end
+end

data/lib/event_tracer/dynamo_db/default_processor.rb

@@ -0,0 +1,15 @@
+module EventTracer
+  module DynamoDB
+    class DefaultProcessor
+      def call(log_type, action:, message:, args:)
+        args.merge(
+          timestamp: Time.now.utc.iso8601(6),
+          action: action,
+          message: message,
+          log_type: log_type,
+          app: EventTracer::Config.config.app_name
+        )
+      end
+    end
+  end
+end

data/lib/event_tracer/dynamo_db/logger.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'time'
+require_relative 'client'
+require_relative 'worker'
+require_relative 'default_processor'
+
+module EventTracer
+  module DynamoDB
+    class Logger < BufferedLogger
+      def initialize(buffer: Buffer.new(buffer_size: 0), log_processor: DefaultProcessor.new)
+        super(buffer: buffer, log_processor: log_processor, worker: Worker)
+      end
+    end
+  end
+end

data/lib/event_tracer/dynamo_db/worker.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+begin
+  require 'sidekiq'
+  require 'aws-sdk-dynamodb'
+rescue LoadError => e
+  puts "Please add the missing gem into your app Gemfile: #{e.message}"
+  raise
+end
+
+module EventTracer
+  module DynamoDB
+    class Worker
+      include ::Sidekiq::Worker
+
+      sidekiq_options retry: 1, queue: EventTracer::Config.config.dynamo_db_queue_name
+
+      # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method
+      MAX_DYNAMO_DB_ITEM_PER_REQUEST = 25
+
+      def initialize(client = nil)
+        @config = EventTracer::Config.config
+        @client = client || @config.dynamo_db_client || Client.call
+      end
+
+      def perform(items)
+        wrap(items).each_slice(MAX_DYNAMO_DB_ITEM_PER_REQUEST) do |batch|
+          data = batch.map do |item|
+            { put_request: { item: clean_empty_values(item) } }
+          end
+
+          client.batch_write_item(
+            request_items: { config.dynamo_db_table_name => data }
+          )
+
+        rescue Aws::DynamoDB::Errors::ServiceError => e
+          EventTracer.error(
+            loggers: %i(base),
+            action: 'DynamoDBWorker',
+            app: EventTracer::Config.config.app_name,
+            error: e.class.name,
+            message: e.message
+          )
+        end
+      end
+
+      private
+
+        attr_reader :client, :config
+
+        def wrap(items)
+          # NOTE: This allows us to handle either buffered or unbuffered payloads
+          if items.is_a?(Hash)
+            [items]
+          else
+            Array(items)
+          end
+        end
+
+        # dynamo can't serialise empty strings/ non-zero numerics
+        def clean_empty_values(data)
+          data.delete_if do |_key, value|
+            case value
+            when Hash
+              clean_empty_values(value)
+              false
+            when String then value.empty?
+            else false
+            end
+          end
+        end
+
+    end
+  end
+end

data/lib/event_tracer/version.rb

@@ -1,3 +1,3 @@
 module EventTracer
-  VERSION = '0.3.1'.freeze
+  VERSION = '0.4.2'.freeze
 end

data/lib/event_tracer.rb

@@ -3,7 +3,7 @@ require 'event_tracer/log_result'
 
 module EventTracer
 
-  LOG_TYPES ||= %i(info warn error)
+  LOG_TYPES = %i(info warn error)
 
   @loggers = {}
 
@@ -70,4 +70,4 @@ module EventTracer
 end
 
 project_root = File.dirname(File.absolute_path(__FILE__))
-Dir.glob("#{project_root}/event_tracer/*") {|file| require file}
+Dir.glob("#{project_root}/event_tracer/*.rb") {|file| require file}

metadata

@@ -1,15 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: event_tracer
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.2
 platform: ruby
 authors:
 - melvrickgoh
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-19 00:00:00.000000000 Z
+date: 2021-12-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dry-configurable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -53,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.0'
 description: 'Thin wrapper for formatted logging/ metric services to be used as a
-  single service. External service(s) supported: Appsignal'
+  single service. External service(s) supported: Appsignal, Datadog, DynamoDB'
 email:
 - melvrickgoh@hotmail.com
 executables: []
@@ -64,7 +92,14 @@ files:
 - lib/event_tracer/appsignal_logger.rb
 - lib/event_tracer/base_logger.rb
 - lib/event_tracer/basic_decorator.rb
+- lib/event_tracer/buffer.rb
+- lib/event_tracer/buffered_logger.rb
+- lib/event_tracer/config.rb
 - lib/event_tracer/datadog_logger.rb
+- lib/event_tracer/dynamo_db/client.rb
+- lib/event_tracer/dynamo_db/default_processor.rb
+- lib/event_tracer/dynamo_db/logger.rb
+- lib/event_tracer/dynamo_db/worker.rb
 - lib/event_tracer/log_result.rb
 - lib/event_tracer/version.rb
 homepage: https://github.com/melvrickgoh/event_tracer