logdna 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61fb8e8caddc3258145cc8d0d6f608822b3afca192012c6798e3cf6924b12cd6
-  data.tar.gz: 5c60cab47160976899feef07a87c0e79fc196c6931175db12e9becaaf3fa7ac4
+  metadata.gz: 32488a458ed8004dcb65531121c286ae1df1b5b3586cf146bb5e973e24e0f559
+  data.tar.gz: 03f487fbff81de61177296ec51849659c66391d7f16a00a2d170e7d03971eaf9
 SHA512:
-  metadata.gz: 0f4eb111244e049c6ae038a98509bad947eb9fd25ebd5dd51786af64881eca54a08e9f8aaa88e58533e09faaafffd7374b3db64ddfadb17cbe14e88c6a0a7429
-  data.tar.gz: 6da73271847ed79db1304730e6749e9d19bde782fa7924420d669e934039fb3f78b1f71a48df712abb64c1e6bf140c8beb7b3018c625370cb3eec12e9e3c4df0
+  metadata.gz: 5a525a02bc844af91ecc52e324ebc9c1323c26ff0fada9cfb32092b9d57f84b7c8dacd7239e8ffb4d36caca3b89533b0fac577a17e3264643c2678368b7abc10
+  data.tar.gz: fa9721cf9605ab44e08d6222a3d8a222c0c0b3eca7802db673ab7a2c3c8bc840ddb060197a39ddb7b5aa149b93fc26d197c09c7c10a54e84f4579d00f7b8ed61

data/README.md

@@ -123,8 +123,12 @@ Instantiates a new instance of the class it is called on. ingestion_key is requi
 |{ :env => STAGING, PRODUCTION .. etc} | Nil |
 |{ :meta => metadata} | Nil |
 |{ :endpoint => LogDNA Ingestion URI | 'https://logs.logdna.com/logs/ingest' |
-|{ :flushtime => Log flush interval in seconds } | 0.25 seconds |
-|{ :flushbyte => Log flush upper limit in bytes } | 500000 bytes ~= 0.5 megabytes |
+|{ :flush_interval => Limit to trigger a flush in seconds } | 0.25 seconds |
+|{ :flush_size => Limit to trigger a flush in bytes } | 2097152 bytes = 2 MiB |
+|{ :request_size => Upper limit of request in bytes } | 2097152 bytes = 2 MiB |
+|{ :retry_timeout => Base timeout for retries in seconds } | 0.25 seconds |
+|{ :retry_max_attempts => Maximum number of retries per request } | 3 attempts |
+|{ :retry_max_jitter => Maximum amount of jitter to add to each retry request in seconds } | 0.25 seconds |
 
 Different log level displays log messages in different colors as well.
 - ![TRACE DEBUG INFO Colors](https://placehold.it/15/515151/000000?text=+)   "Trace"  "Debug"  "Info"

data/lib/logdna.rb

@@ -3,12 +3,13 @@
 require "logger"
 require "socket"
 require "uri"
-require_relative "logdna/client.rb"
-require_relative "logdna/resources.rb"
-require_relative "logdna/version.rb"
+require_relative "logdna/client"
+require_relative "logdna/resources"
+require_relative "logdna/version"
 
 module Logdna
   class ValidURLRequired < ArgumentError; end
+
   class MaxLengthExceeded < ArgumentError; end
 
   class Ruby < ::Logger
@@ -18,11 +19,12 @@ module Logdna
     attr_accessor :app, :env, :meta
 
     def initialize(key, opts = {})
+      super(nil, nil, nil)
       @app = opts[:app] || "default"
       @log_level = opts[:level] || "INFO"
       @env = opts[:env]
       @meta = opts[:meta]
-      @internal_logger = Logger.new(STDOUT)
+      @internal_logger = Logger.new($stdout)
       @internal_logger.level = Logger::DEBUG
       endpoint = opts[:endpoint] || Resources::ENDPOINT
       hostname = opts[:hostname] || Socket.gethostname
@@ -127,9 +129,5 @@ module Logdna
     def close
       @client&.exitout
     end
-
-    at_exit do
-      @client&.exitout
-    end
   end
 end

data/lib/logdna/client.rb

@@ -1,34 +1,57 @@
 # frozen_string_literal: true
 
+require "etc"
 require "net/http"
 require "socket"
 require "json"
 require "concurrent"
 require "date"
+require "securerandom"
 
 module Logdna
+  Message = Struct.new(:source, :running_size)
+
   class Client
     def initialize(request, uri, opts)
       @uri = uri
 
       # NOTE: buffer is in memory
       @buffer = []
-      @buffer_byte_size = 0
-
-      @side_messages = []
 
       @lock = Mutex.new
-      @side_message_lock = Mutex.new
-      @flush_limit = opts[:flush_size] || Resources::FLUSH_BYTE_LIMIT
+
       @flush_interval = opts[:flush_interval] || Resources::FLUSH_INTERVAL
-      @flush_scheduled = false
-      @exception_flag = false
+      @flush_size = opts[:flush_size] || Resources::FLUSH_SIZE
 
       @request = request
+      @request_size = opts[:request_size] || Resources::REQUEST_SIZE
+
       @retry_timeout = opts[:retry_timeout] || Resources::RETRY_TIMEOUT
+      @retry_max_jitter = opts[:retry_max_jitter] || Resources::RETRY_MAX_JITTER
+      @retry_max_attempts = opts[:retry_max_attempts] || Resources::RETRY_MAX_ATTEMPTS
 
-      @internal_logger = Logger.new(STDOUT)
+      @internal_logger = Logger.new($stdout)
       @internal_logger.level = Logger::DEBUG
+
+      @work_thread_pool = Concurrent::FixedThreadPool.new(Etc.nprocessors)
+      # TODO: Expose an option to configure the maximum concurrent requests
+      # Requires the instance-global request to be resolved first
+      @request_thread_pool = Concurrent::FixedThreadPool.new(Resources::MAX_CONCURRENT_REQUESTS)
+
+      @scheduled_flush = nil
+    end
+
+    def schedule_flush
+      if @scheduled_flush.nil? || @scheduled_flush.complete?
+        @scheduled_flush = Concurrent::ScheduledTask.execute(@flush_interval) { flush }
+      end
+    end
+
+    def unschedule_flush
+      if !@scheduled_flush.nil?
+        @scheduled_flush.cancel
+        @scheduled_flush = nil
+      end
     end
 
     def process_message(msg, opts = {})
@@ -44,95 +67,145 @@ module Logdna
       processed_message
     end
 
-    def schedule_flush
-      start_timer = lambda {
-        sleep(@exception_flag ? @retry_timeout : @flush_interval)
-        flush if @flush_scheduled
-      }
-      Thread.new { start_timer.call }
+    def write_to_buffer(msg, opts)
+      Concurrent::Future.execute({ executor: @work_thread_pool }) { write_to_buffer_sync(msg, opts) }
     end
 
-    def write_to_buffer(msg, opts)
-      if @lock.try_lock
-        processed_message = process_message(msg, opts)
-        new_message_size = processed_message.to_s.bytesize
-        @buffer.push(processed_message)
-        @buffer_byte_size += new_message_size
-        @flush_scheduled = true
-        @lock.unlock
-
-        if @flush_limit <= @buffer_byte_size
-          flush
-        else
-          schedule_flush
+    def write_to_buffer_sync(msg, opts)
+      processed_message = process_message(msg, opts)
+      message_size = processed_message.to_s.bytesize
+
+      running_size = @lock.synchronize do
+        running_size = message_size
+        if @buffer.any?
+          running_size += @buffer[-1].running_size
         end
+        @buffer.push(Message.new(processed_message, running_size))
+
+        running_size
+      end
+
+      if running_size >= @flush_size
+        unschedule_flush
+        flush_sync
       else
-        @side_message_lock.synchronize do
-          @side_messages.push(process_message(msg, opts))
-        end
+        schedule_flush
       end
     end
 
-    # This method has to be called with @lock
-    def send_request
-      @side_message_lock.synchronize do
-        @buffer.concat(@side_messages)
-        @side_messages.clear
+    ##
+    # Flushes all logs to LogDNA asynchronously
+    def flush(options = {})
+      Concurrent::Future.execute({ executor: @work_thread_pool }) { flush_sync(options) }
+    end
+
+    ##
+    # Flushes all logs to LogDNA synchronously
+    def flush_sync(options = {})
+      slices = @lock.synchronize do
+        # Slice the buffer into chunks that try to be no larger than @request_size. Slice points are found with
+        # a binary search thanks to the structure of @buffer. We are working backwards because it's cheaper to
+        # remove from the tail of an array instead of the head
+        slices = []
+        until @buffer.empty?
+          search_size = @buffer[-1].running_size - @request_size
+          if search_size.negative?
+            search_size = 0
+          end
+
+          slice_index = @buffer.bsearch_index { |message| message.running_size >= search_size }
+          slices.push(@buffer.pop(@buffer.length - slice_index).map(&:source))
+        end
+        slices
+      end
+
+      # Remember the chunks are in reverse order, this un-reverses them
+      slices.reverse_each do |slice|
+        if options[:block_on_requests]
+          try_request(slice)
+        else
+          Concurrent::Future.execute({ executor: @request_thread_pool }) { try_request(slice) }
+        end
       end
+    end
 
-      @request.body = {
+    def try_request(slice)
+      body = {
         e: "ls",
-        ls: @buffer
+        ls: slice
       }.to_json
 
-      handle_exception = lambda do |message|
-        @internal_logger.debug(message)
-        @exception_flag = true
-        @side_message_lock.synchronize do
-          @side_messages.concat(@buffer)
+      flush_id = "#{SecureRandom.uuid} [#{slice.length} lines]"
+      error_header = "Flush {#{flush_id}} failed."
+      tries = 0
+      loop do
+        tries += 1
+
+        if tries > @retry_max_attempts
+          @internal_logger.debug("Flush {#{flush_id}} exceeded 3 tries. Discarding flush buffer")
+          break
         end
+
+        if send_request(body, error_header)
+          break
+        end
+
+        sleep(@retry_timeout * (1 << (tries - 1)) + rand(@retry_max_jitter))
       end
+    end
 
+    def send_request(body, error_header)
+      # TODO: Remove instance-global request object
+      @request.body = body
       begin
-        @response = Net::HTTP.start(
+        response = Net::HTTP.start(
           @uri.hostname,
           @uri.port,
           use_ssl: @uri.scheme == "https"
         ) do |http|
           http.request(@request)
         end
-        if @response.is_a?(Net::HTTPForbidden)
-          @internal_logger.debug("Please provide a valid ingestion key")
-        elsif !@response.is_a?(Net::HTTPSuccess)
-          handle_exception.call("The response is not successful #{@response}")
+
+        code = response.code.to_i
+        if [401, 403].include?(code)
+          @internal_logger.debug("#{error_header} Please provide a valid ingestion key. Discarding flush buffer")
+          return true
+        elsif [408, 500, 504].include?(code)
+          # These codes might indicate a temporary ingester issue
+          @internal_logger.debug("#{error_header} The request failed #{response}. Retrying")
+        elsif code == 200
+          return true
+        else
+          @internal_logger.debug("#{error_header} The request failed #{response}. Discarding flush buffer")
+          return true
         end
-        @exception_flag = false
       rescue SocketError
-        handle_exception.call("Network connectivity issue")
+        @internal_logger.debug("#{error_header} Network connectivity issue. Retrying")
       rescue Errno::ECONNREFUSED => e
-        handle_exception.call("The server is down. #{e.message}")
+        @internal_logger.debug("#{error_header} The server is down. #{e.message}. Retrying")
       rescue Timeout::Error => e
-        handle_exception.call("Timeout error occurred. #{e.message}")
-      ensure
-        @buffer.clear
+        @internal_logger.debug("#{error_header} Timeout error occurred. #{e.message}. Retrying")
       end
-    end
 
-    def flush
-      if @lock.try_lock
-        @flush_scheduled = false
-        if @buffer.any? || @side_messages.any?
-          send_request
-        end
-        @lock.unlock
-      else
-        schedule_flush
-      end
+      false
     end
 
     def exitout
-      flush
-      @internal_logger.debug("Exiting LogDNA logger: Logging remaining messages")
+      unschedule_flush
+      @work_thread_pool.shutdown
+      if !@work_thread_pool.wait_for_termination(1)
+        @internal_logger.warn("Work thread pool unable to shutdown gracefully. Logs potentially dropped")
+      end
+      @request_thread_pool.shutdown
+      if !@request_thread_pool.wait_for_termination(5)
+        @internal_logger.warn("Request thread pool unable to shutdown gracefully. Logs potentially dropped")
+      end
+
+      if @buffer.any?
+        @internal_logger.debug("Exiting LogDNA logger: Logging remaining messages")
+        flush_sync({ block_on_requests: true })
+        @internal_logger.debug("Finished flushing logs to LogDNA")
+      end
     end
   end
 end

data/lib/logdna/resources.rb

@@ -8,10 +8,14 @@ module Resources
   MAX_REQUEST_TIMEOUT = 300_000
   MAX_LINE_LENGTH = 32_000
   MAX_INPUT_LENGTH = 80
-  RETRY_TIMEOUT = 60
+  RETRY_TIMEOUT = 0.25
+  RETRY_MAX_ATTEMPTS = 3
+  RETRY_MAX_JITTER = 0.5
   FLUSH_INTERVAL = 0.25
-  FLUSH_BYTE_LIMIT = 500_000
+  FLUSH_SIZE = 2 * 1_024 * 1_024
+  REQUEST_SIZE = 2 * 1_024 * 1_024
   ENDPOINT = "https://logs.logdna.com/logs/ingest"
   MAC_ADDR_CHECK = /^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$/.freeze
   IP_ADDR_CHECK = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.freeze
+  MAX_CONCURRENT_REQUESTS = 1
 end

data/lib/logdna/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LogDNA
-  VERSION = "1.4.2"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logdna
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Gun Woo Choi, Derek Zhou, Vilya Levitskiy, Muaz Siddiqui
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-19 00:00:00.000000000 Z
+date: 2021-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -90,7 +90,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="