hyperdx-ruby 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 4933bb0e69a1ad181315c58d286df867e1c657f3571fae7b1deeda5d731ca630
+  data.tar.gz: 97bbb1208f0052283abee07c27a04a0005590e3dd06097e62644f04d86857014
+SHA512:
+  metadata.gz: b8fab542413e3d0ff9fd02f58356e0ae238bf37fe2e235052b02b735b64d5332dd415777571228dcf24ae5de2755ec9c0f2fbc991984bba780d62f07ee12eb27
+  data.tar.gz: 357382ef8d18d2dbdf412c7595d85cfc48d5e287d719a5b34210bc9989f2a56672cc54f7f951bf2e071503c8d1b23558f778cf2963380915a03d752572c2383f

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 TODO: Write your name
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Hyperdx::Ruby
+
+TODO: Delete this and the text below, and describe your gem
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/hyperdx/ruby`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/hyperdx-ruby. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/hyperdx-ruby/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Hyperdx::Ruby project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/hyperdx-ruby/blob/main/CODE_OF_CONDUCT.md).

data/lib/hyperdx/ruby/client.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "etc"
+require "net/http"
+require "socket"
+require "json"
+require "concurrent"
+require "date"
+require "securerandom"
+
+module Hyperdx
+  Message = Struct.new(:source, :running_size)
+
+  class Client
+    def initialize(request, uri, opts)
+      @uri = uri
+
+      # NOTE: buffer is in memory
+      @buffer = []
+
+      @lock = Mutex.new
+
+      @flush_interval = opts[:flush_interval] || Resources::FLUSH_INTERVAL
+      @flush_size = opts[:flush_size] || Resources::FLUSH_SIZE
+
+      @request = request
+      @request_size = opts[:request_size] || Resources::REQUEST_SIZE
+
+      @retry_timeout = opts[:retry_timeout] || Resources::RETRY_TIMEOUT
+      @retry_max_jitter = opts[:retry_max_jitter] || Resources::RETRY_MAX_JITTER
+      @retry_max_attempts = opts[:retry_max_attempts] || Resources::RETRY_MAX_ATTEMPTS
+
+      @internal_logger = Logger.new($stdout)
+      @internal_logger.level = Logger::DEBUG
+
+      @work_thread_pool = Concurrent::FixedThreadPool.new(Etc.nprocessors)
+      # TODO: Expose an option to configure the maximum concurrent requests
+      # Requires the instance-global request to be resolved first
+      @request_thread_pool = Concurrent::FixedThreadPool.new(Resources::MAX_CONCURRENT_REQUESTS)
+
+      @scheduled_flush = nil
+    end
+
+    def schedule_flush
+      if @scheduled_flush.nil? || @scheduled_flush.complete?
+        @scheduled_flush = Concurrent::ScheduledTask.execute(@flush_interval) { flush }
+      end
+    end
+
+    def unschedule_flush
+      if !@scheduled_flush.nil?
+        @scheduled_flush.cancel
+        @scheduled_flush = nil
+      end
+    end
+
+    def process_message(msg, opts = {})
+      processed_message = {
+        line: msg,
+        app: opts[:app],
+        level: opts[:level],
+        env: opts[:env],
+        meta: opts[:meta],
+        timestamp: Time.now.to_i,
+      }
+      processed_message.delete(:meta) if processed_message[:meta].nil?
+      processed_message
+    end
+
+    def write_to_buffer(msg, opts)
+      Concurrent::Future.execute({ executor: @work_thread_pool }) { write_to_buffer_sync(msg, opts) }
+    end
+
+    def write_to_buffer_sync(msg, opts)
+      processed_message = process_message(msg, opts)
+      message_size = processed_message.to_s.bytesize
+
+      running_size = @lock.synchronize do
+        running_size = message_size
+        if @buffer.any?
+          running_size += @buffer[-1].running_size
+        end
+        @buffer.push(Message.new(processed_message, running_size))
+
+        running_size
+      end
+
+      if running_size >= @flush_size
+        unschedule_flush
+        flush_sync
+      else
+        schedule_flush
+      end
+    end
+
+    ##
+    # Flushes all logs to HyperDX asynchronously
+    def flush(options = {})
+      Concurrent::Future.execute({ executor: @work_thread_pool }) { flush_sync(options) }
+    end
+
+    ##
+    # Flushes all logs to HyperDX synchronously
+    def flush_sync(options = {})
+      slices = @lock.synchronize do
+        # Slice the buffer into chunks that try to be no larger than @request_size. Slice points are found with
+        # a binary search thanks to the structure of @buffer. We are working backwards because it's cheaper to
+        # remove from the tail of an array instead of the head
+        slices = []
+        until @buffer.empty?
+          search_size = @buffer[-1].running_size - @request_size
+          if search_size.negative?
+            search_size = 0
+          end
+
+          slice_index = @buffer.bsearch_index { |message| message.running_size >= search_size }
+          slices.push(@buffer.pop(@buffer.length - slice_index).map(&:source))
+        end
+        slices
+      end
+
+      # Remember the chunks are in reverse order, this un-reverses them
+      slices.reverse_each do |slice|
+        if options[:block_on_requests]
+          try_request(slice)
+        else
+          Concurrent::Future.execute({ executor: @request_thread_pool }) { try_request(slice) }
+        end
+      end
+    end
+
+    def try_request(slice)
+      body = slice.to_json
+
+      flush_id = "#{SecureRandom.uuid} [#{slice.length} lines]"
+      error_header = "Flush {#{flush_id}} failed."
+      tries = 0
+      loop do
+        tries += 1
+
+        if tries > @retry_max_attempts
+          @internal_logger.debug("Flush {#{flush_id}} exceeded 3 tries. Discarding flush buffer")
+          break
+        end
+
+        if send_request(body, error_header)
+          break
+        end
+
+        sleep(@retry_timeout * (1 << (tries - 1)) + rand(@retry_max_jitter))
+      end
+    end
+
+    def send_request(body, error_header)
+      # TODO: Remove instance-global request object
+      @request.body = body
+      begin
+        response = Net::HTTP.start(
+          @uri.hostname,
+          @uri.port,
+          use_ssl: @uri.scheme == "https"
+        ) do |http|
+          http.request(@request)
+        end
+
+        code = response.code.to_i
+        if [401, 403].include?(code)
+          @internal_logger.debug("#{error_header} Please provide a valid ingestion key. Discarding flush buffer")
+          return true
+        elsif [408, 500, 504].include?(code)
+          # These codes might indicate a temporary ingester issue
+          @internal_logger.debug("#{error_header} The request failed #{response}. Retrying")
+        elsif code == 200
+          return true
+        else
+          @internal_logger.debug("#{error_header} The request failed #{response}. Discarding flush buffer")
+          return true
+        end
+      rescue SocketError
+        @internal_logger.debug("#{error_header} Network connectivity issue. Retrying")
+      rescue Errno::ECONNREFUSED => e
+        @internal_logger.debug("#{error_header} The server is down. #{e.message}. Retrying")
+      rescue Timeout::Error => e
+        @internal_logger.debug("#{error_header} Timeout error occurred. #{e.message}. Retrying")
+      end
+
+      false
+    end
+
+    def exitout
+      unschedule_flush
+      @work_thread_pool.shutdown
+      if !@work_thread_pool.wait_for_termination(1)
+        @internal_logger.warn("Work thread pool unable to shutdown gracefully. Logs potentially dropped")
+      end
+      @request_thread_pool.shutdown
+      if !@request_thread_pool.wait_for_termination(5)
+        @internal_logger.warn("Request thread pool unable to shutdown gracefully. Logs potentially dropped")
+      end
+
+      if @buffer.any?
+        @internal_logger.debug("Exiting HyperDX logger: Logging remaining messages")
+        flush_sync({ block_on_requests: true })
+        @internal_logger.debug("Finished flushing logs to HyperDX")
+      end
+    end
+  end
+end

data/lib/hyperdx/ruby/resources.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Resources
+  LOG_LEVELS = %w[DEBUG INFO WARN ERROR FATAL TRACE].freeze
+  DEFAULT_REQUEST_HEADER = { "Content-Type" => "application/json; charset=UTF-8" }.freeze
+  DEFAULT_REQUEST_TIMEOUT = 180_000
+  MS_IN_A_DAY = 86_400_000
+  MAX_REQUEST_TIMEOUT = 300_000
+  MAX_LINE_LENGTH = 32_000
+  MAX_INPUT_LENGTH = 80
+  RETRY_TIMEOUT = 0.25
+  RETRY_MAX_ATTEMPTS = 3
+  RETRY_MAX_JITTER = 0.5
+  FLUSH_INTERVAL = 0.25
+  FLUSH_SIZE = 2 * 1_024 * 1_024
+  REQUEST_SIZE = 2 * 1_024 * 1_024
+  ENDPOINT = "https://in.hyperdx.io"
+  MAC_ADDR_CHECK = /^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$/.freeze
+  IP_ADDR_CHECK = /^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$/.freeze
+  MAX_CONCURRENT_REQUESTS = 1
+end

data/lib/hyperdx/ruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Hyperdx
+  VERSION = "0.1.0"
+end

data/lib/hyperdx/ruby.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require "logger"
+require "socket"
+require "uri"
+require_relative "ruby/client"
+require_relative "ruby/resources"
+require_relative "ruby/version"
+
+module Hyperdx
+  class ValidURLRequired < ArgumentError; end
+
+  class MaxLengthExceeded < ArgumentError; end
+
+  class Ruby < ::Logger
+    # uncomment line below and line 3 to enforce singleton
+    # include Singleton
+    Logger::TRACE = 5
+    attr_accessor :app, :env, :meta
+
+    def initialize(key, opts = {})
+      super(nil, nil, nil)
+      @app = opts[:app] || "default"
+      @log_level = opts[:level] || "INFO"
+      @env = opts[:env]
+      @meta = opts[:meta]
+      @internal_logger = Logger.new($stdout)
+      @internal_logger.level = Logger::DEBUG
+      endpoint = opts[:endpoint] || Resources::ENDPOINT
+      hostname = opts[:hostname] || Socket.gethostname
+
+      if hostname.size > Resources::MAX_INPUT_LENGTH || @app.size > Resources::MAX_INPUT_LENGTH
+        @internal_logger.debug("Hostname or Appname is over #{Resources::MAX_INPUT_LENGTH} characters")
+        return
+      end
+
+      ip =  opts.key?(:ip) ? "&ip=#{opts[:ip]}" : ""
+      mac = opts.key?(:mac) ? "&mac=#{opts[:mac]}" : ""
+      url = "#{endpoint}?hostname=#{hostname}#{mac}#{ip}"
+      uri = URI(url)
+
+      request = Net::HTTP::Post.new(uri.request_uri, "Content-Type" => "application/json")
+      request['Authorization'] = "Bearer #{key}"
+      request[:'user-agent'] = opts[:'user-agent'] || "ruby/#{Hyperdx::VERSION}"
+      @client = Hyperdx::Client.new(request, uri, opts)
+    end
+
+    def default_opts
+      {
+        app: @app,
+        level: @log_level,
+        env: @env,
+        meta: @meta,
+      }
+    end
+
+    def level
+      @log_level
+    end
+
+    def level=(value)
+      if value.is_a? Numeric
+        @log_level = Resources::LOG_LEVELS[value]
+        return
+      end
+
+      @log_level = value
+    end
+
+    def log(message = nil, opts = {})
+      if message.nil? && block_given?
+        message = yield
+      end
+      if message.nil?
+        @internal_logger.debug("provide either a message or block")
+        return
+      end
+      message = message.to_s.encode("UTF-8")
+      @client.write_to_buffer(message, default_opts.merge(opts).merge(
+                                         timestamp: (Time.now.to_f * 1000).to_i
+                                       ))
+    end
+
+    Resources::LOG_LEVELS.each do |lvl|
+      name = lvl.downcase
+
+      define_method name do |msg = nil, opts = {}, &block|
+        self.log(msg, opts.merge(
+                        level: lvl
+                      ), &block)
+      end
+
+      define_method "#{name}?" do
+        return Resources::LOG_LEVELS[self.level] == lvl if level.is_a? Numeric
+
+        self.level == lvl
+      end
+    end
+
+    def clear
+      @app = "default"
+      @log_level = "INFO"
+      @env = nil
+      @meta = nil
+    end
+
+    def <<(msg = nil, opts = {})
+      log(msg, opts.merge(
+                 level: ""
+               ))
+    end
+
+    def add(*_arg)
+      @internal_logger.debug("add not supported in HyperDX logger")
+      false
+    end
+
+    def unknown(msg = nil, opts = {})
+      log(msg, opts.merge(
+                 level: "UNKNOWN"
+               ))
+    end
+
+    def datetime_format(*_arg)
+      @internal_logger.debug("datetime_format not supported in HyperDX logger")
+      false
+    end
+
+    def close
+      @client&.exitout
+    end
+  end
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: hyperdx-ruby
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Warren Lee
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-05-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: require_all
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.18'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.18'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.78'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.78'
+description: ''
+email:
+- warren@hyperdx.io
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/hyperdx/ruby.rb
+- lib/hyperdx/ruby/client.rb
+- lib/hyperdx/ruby/resources.rb
+- lib/hyperdx/ruby/version.rb
+homepage: https://github.com/hyperdxio/hyperdx-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.5.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: HyperDX Ruby SDK
+test_files: []