sillydev 1.0.0

data/lib/sillydev/monitoring/client.rb

@@ -0,0 +1,674 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "securerandom"
+require "stringio"
+require "uri"
+require "zlib"
+require_relative "aggregator"
+require_relative "runtime"
+require_relative "serialize"
+
+module SillyDev
+  module Monitoring
+    # One configured monitoring client: buffers, aggregates and sends to a Feathers ingest node.
+    # Use the functions on SillyDev::Monitoring instead.
+    #
+    # Nothing here runs on require: a client only starts its thread when Monitoring.init makes one
+    # with a key. Its public methods never raise into application code.
+    class Client
+      MAX_ITEMS_PER_REQUEST = 500
+      MAX_SLOW_PER_REQUEST = 10
+      MAX_SLOW_PER_WINDOW = 10
+      MAX_RAW_BYTES = 1000 * 1024 # under Feathers' 1 MiB decompressed limit
+      MAX_SENT_BYTES = 250 * 1024 # under Feathers' 256 KiB as-sent limit
+      GZIP_MIN_BYTES = 1024
+      REQUEST_TIMEOUT = 5.0
+      STALE_AFTER = 55 * 60 # Feathers drops items over an hour old
+      BACKOFF_BASE = 1.0
+      BACKOFF_CAP = 300.0
+
+      Pending = Struct.new(:array, :item, :size) # rubocop:disable Lint/StructNewOverride
+
+      attr_reader :key, :endpoint, :service, :environment, :release, :instance, :sample_rate,
+                  :flush_interval, :max_buffer_bytes, :slow_threshold_ms, :dropped
+      attr_accessor :clock, :monotonic_clock, :random, :transport
+
+      def self.runtime_name
+        engine = defined?(RUBY_ENGINE) ? RUBY_ENGINE : "ruby"
+        version = if engine == "ruby"
+                    RUBY_VERSION
+                  else
+                    (defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION)
+                  end
+        "#{engine} #{version}"
+      end
+
+      def initialize(key = nil, endpoint: nil, service: nil, environment: nil, release: nil, runtime_metrics: true,
+                     capture_unhandled: true, sample_rate: 1.0, before_send: nil, flush_interval: 10.0,
+                     max_buffer_bytes: 1_000_000, slow_threshold_ms: 1000, logger: nil, debug: false, active: true)
+        @key = key
+        @endpoint = endpoint.to_s.chomp("/")
+        @service = (service || "default").to_s.downcase[0, 64]
+        @environment = (environment || "production").to_s.downcase[0, 64]
+        @release = release ? release.to_s[0, 64] : nil
+        @runtime_metrics = runtime_metrics
+        @capture_unhandled = capture_unhandled
+        @sample_rate = sample_rate.to_f.clamp(0.0, 1.0)
+        @before_send = before_send
+        @flush_interval = [flush_interval.to_f, 0.05].max
+        @max_buffer_bytes = [max_buffer_bytes.to_i, 1024].max
+        @slow_threshold_ms = slow_threshold_ms.to_f
+        @logger = logger
+        @debug = debug
+        @instance = SecureRandom.hex(4)
+        @active = active && key && !@endpoint.empty? ? true : false
+
+        # Replaceable in tests.
+        @clock = -> { Time.now.to_f }
+        @monotonic_clock = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+        @random = -> { rand }
+        @transport = nil # ->(url, body, headers, timeout) { [status, headers, body] }
+
+        reset_state
+        @logged = {}
+        @stopped = false # by a 401/403: never sends again
+        @closed = false
+      end
+
+      def inspect
+        "#<#{self.class.name} endpoint=#{@endpoint.inspect} service=#{@service.inspect} enabled=#{enabled?}>"
+      end
+
+      def runtime_metrics?
+        @runtime_metrics
+      end
+
+      def capture_unhandled?
+        @capture_unhandled
+      end
+
+      def active?
+        @active
+      end
+
+      # True while the client has a key, has not been stopped by the ingest and is not shut down.
+      def enabled?
+        @active && !@stopped && !@closed
+      end
+
+      # Starts the thread (called by Monitoring.init).
+      def start
+        return if !@active || @closed
+
+        @runtime = RuntimeCollector.new if @runtime_metrics
+        start_thread
+        debug_log("sending to #{@endpoint} as service #{@service.inspect}, environment " \
+                  "#{@environment.inspect}, instance #{@instance}")
+      end
+
+      def shutdown(timeout = 2.0)
+        return if @closed
+
+        deadline = now_mono + [timeout.to_f, 0.0].max
+        @stop = true
+        wake
+        flush_until(deadline) if @active && !@stopped
+        @closed = true
+        thread = @thread
+        thread.join((deadline - now_mono).clamp(0, 0.5)) if thread && thread != Thread.current
+        nil
+      rescue StandardError
+        nil
+      end
+
+      # Resets everything that belongs to the parent process and starts again (after fork).
+      def after_fork
+        reset_state
+        @instance = SecureRandom.hex(4)
+        @runtime&.reset
+        start_thread if enabled?
+      end
+
+      # -- recording ------------------------------------------------------------------------------
+
+      def increment(name, value = 1, tags = nil)
+        metric("counter", name, value, tags)
+      end
+
+      def gauge(name, value, tags = nil)
+        metric("gauge", name, value, tags)
+      end
+
+      def timing(name, ms, tags = nil)
+        metric("histogram", name, ms, tags)
+      end
+
+      def event(name, attributes = nil)
+        safely do
+          next unless enabled?
+          next log_once("event:#{name}", :warn, "ignoring event with invalid name #{name.inspect}") unless Serialize.valid_name?(name)
+
+          item = { "ts" => @clock.call, "name" => name,
+                   "attributes" => Serialize.bounded_object(attributes, Serialize::MAX_ATTRIBUTES_BYTES) }
+          submit("events", "event", item, sampled: true)
+        end
+      end
+
+      def capture_exception(exception, tags: nil, context: nil, fingerprint: nil, handled: true)
+        safely do
+          next unless enabled? && exception.is_a?(Exception)
+          next if exception.instance_variable_defined?(:@__sillydev_monitoring_captured)
+
+          begin
+            exception.instance_variable_set(:@__sillydev_monitoring_captured, true)
+          rescue FrozenError
+            nil
+          end
+          body = Serialize.serialize_exception(exception)
+          item = {
+            "ts" => @clock.call,
+            "type" => body["type"],
+            "message" => body["message"],
+            "handled" => handled ? true : false,
+            "stack" => body["stack"],
+            "fingerprint" => fingerprint ? fingerprint.to_s[0, Serialize::MAX_FINGERPRINT] : nil,
+            "tags" => Serialize.clean_tags(tags).to_h,
+            "context" => Serialize.bounded_object(context, Serialize::MAX_CONTEXT_BYTES),
+          }
+          item["cause"] = body["cause"] if body.key?("cause")
+          submit("errors", "error", item, sampled: true)
+        end
+      end
+
+      def capture_message(message, level: "error", tags: nil, context: nil)
+        safely do
+          next unless enabled?
+
+          level = (level || "error").to_s.downcase
+          all_tags = (tags || {}).transform_keys(&:to_s)
+          all_tags["level"] ||= level
+          item = {
+            "ts" => @clock.call,
+            "type" => "Message",
+            "message" => message.to_s[0, Serialize::MAX_MESSAGE],
+            "handled" => true,
+            "stack" => [],
+            "fingerprint" => nil,
+            "tags" => Serialize.clean_tags(all_tags).to_h,
+            "context" => Serialize.bounded_object(context, Serialize::MAX_CONTEXT_BYTES),
+          }
+          submit("errors", "error", item, sampled: true)
+        end
+      end
+
+      def record_slow(name, duration_ms, kind: "custom", tags: nil, attributes: nil)
+        safely do
+          next unless enabled?
+
+          duration = finite(duration_ms)
+          next if duration.nil?
+
+          item = {
+            "ts" => @clock.call,
+            "name" => name.to_s[0, 256],
+            "kind" => Serialize::SLOW_KINDS.include?(kind.to_s) ? kind.to_s : "custom",
+            "duration_ms" => [duration, 0.0].max.round(3),
+            "tags" => Serialize.clean_tags(tags).to_h,
+            "attributes" => Serialize.bounded_object(attributes, Serialize::MAX_ATTRIBUTES_BYTES),
+          }
+          item = filter("slow", item, sampled: false)
+          next if item.nil?
+
+          @lock.synchronize do
+            @slow << item
+            if @slow.size > 1000
+              @slow.sort_by! { |s| -s["duration_ms"] }
+              count_drop("slow_buffer", @slow.size - MAX_SLOW_PER_WINDOW)
+              @slow = @slow.first(MAX_SLOW_PER_WINDOW)
+            end
+          end
+        end
+      end
+
+      # Marks this instance alive now (sent straight away by the background thread).
+      def heartbeat
+        safely do
+          next unless enabled?
+
+          @lock.synchronize { @heartbeat_due = true }
+          wake
+        end
+      end
+
+      # Records a slow operation when duration_ms is over the threshold.
+      def maybe_slow(name, duration_ms, kind, tags = nil)
+        record_slow(name, duration_ms, kind: kind, tags: tags) if enabled? && duration_ms >= @slow_threshold_ms
+      end
+
+      # Sends everything buffered, including the current window. True when nothing is left.
+      def flush(timeout = 2.0)
+        safely(false) do
+          next false unless enabled?
+
+          flush_until(now_mono + [timeout.to_f, 0.0].max)
+        end
+      end
+
+      # -- for tests and the integrations ---------------------------------------------------------
+
+      def pending_items
+        @lock.synchronize { @pending.map { |p| [p.array, p.item] } }
+      end
+
+      private
+
+      def reset_state
+        @lock = Mutex.new
+        @send_lock = Mutex.new
+        @wake_lock = Mutex.new
+        @wake_cond = ConditionVariable.new
+        @woken = false
+        @stop = false
+        @aggregator = Aggregator.new
+        @pending = []
+        @pending_bytes = 0
+        @slow = []
+        @heartbeat_due = false
+        @dropped = Hash.new(0)
+        @failures = 0
+        @next_attempt = 0.0
+        @thread = nil
+      end
+
+      def now_mono
+        @monotonic_clock.call
+      end
+
+      def safely(fallback = nil)
+        yield
+      rescue StandardError => e
+        debug_log("#{e.class}: #{e.message}")
+        fallback
+      end
+
+      def finite(value)
+        number = Float(value)
+        number.finite? ? number : nil
+      rescue ArgumentError, TypeError
+        nil
+      end
+
+      def metric(kind, name, value, tags)
+        safely do
+          next unless enabled?
+          next log_once("name:#{name}", :warn, "ignoring metric with invalid name #{name.inspect}") unless Serialize.valid_name?(name)
+
+          number = finite(value)
+          if number.nil? || (kind != "gauge" && number.negative?)
+            next unless kind == "histogram" && number
+
+            number = 0.0
+          end
+          key = Serialize.clean_tags(tags)
+          now = @clock.call
+          @lock.synchronize do
+            case kind
+            when "counter" then @aggregator.counter(now, name, number, key)
+            when "gauge" then @aggregator.gauge(now, name, number, key)
+            else @aggregator.histogram(now, name, number, key)
+            end
+          end
+        end
+      end
+
+      def filter(item_type, item, sampled:)
+        return nil if sampled && @sample_rate < 1.0 && @random.call >= @sample_rate
+        return item if @before_send.nil?
+
+        begin
+          result = @before_send.call(item_type, item)
+        rescue StandardError
+          log_once("before_send", :warn, "before_send raised; the item was dropped")
+          return nil
+        end
+        result.is_a?(Hash) ? result : nil
+      end
+
+      def submit(array, item_type, item, sampled:)
+        filtered = filter(item_type, item, sampled: sampled)
+        return if filtered.nil?
+
+        full = @lock.synchronize do
+          push(array, filtered)
+          @pending_bytes >= MAX_SENT_BYTES * 0.75 || @pending.size >= MAX_ITEMS_PER_REQUEST
+        end
+        wake if full
+      end
+
+      # Appends to the pending buffer, dropping the oldest items over the size bound. Lock held.
+      def push(array, item)
+        size = JSON.generate(item).bytesize
+        @pending << Pending.new(array, item, size)
+        @pending_bytes += size
+        enforce_bound
+      end
+
+      def enforce_bound
+        while @pending_bytes > @max_buffer_bytes && !@pending.empty?
+          @pending_bytes -= @pending.shift.size
+          count_drop("buffer_full")
+          log_once("buffer_full", :warn, "buffer full, dropping the oldest data")
+        end
+      end
+
+      def count_drop(reason, count = 1)
+        @dropped[reason] += count
+      end
+
+      # Moves ended windows (all of them when final) and the slowest operations into pending.
+      def move_ready(final)
+        @lock.synchronize do
+          @aggregator.drain(final ? nil : @clock.call).each { |point| push("metrics", point) }
+          if @aggregator.dropped.positive?
+            count_drop("series_buffer", @aggregator.dropped)
+            log_once("series_buffer", :warn, "too many distinct series, dropping new ones")
+            @aggregator.dropped = 0
+          end
+          unless @slow.empty?
+            @slow.sort_by! { |s| -s["duration_ms"] }
+            count_drop("slow_window", @slow.size - MAX_SLOW_PER_WINDOW) if @slow.size > MAX_SLOW_PER_WINDOW
+            @slow.first(MAX_SLOW_PER_WINDOW).each { |item| push("slow", item) }
+            @slow = []
+          end
+        end
+      end
+
+      def collect
+        @runtime&.collect&.each { |name, kind, value| metric(kind, name, value, nil) }
+        Monitoring.collectors.dup.each do |collector|
+          collector.call(self)
+        rescue StandardError => e
+          log_once("collector:#{collector.object_id}", :warn, "a metrics collector failed: #{e.class}: #{e.message}")
+        end
+      end
+
+      def take_batch
+        @lock.synchronize do
+          heartbeat = @heartbeat_due || (@runtime_metrics && !@pending.empty?)
+          @heartbeat_due = false
+          batch = []
+          skipped = []
+          slow = 0
+          oldest = @clock.call - STALE_AFTER
+          while !@pending.empty? && batch.size < MAX_ITEMS_PER_REQUEST
+            entry = @pending.shift
+            @pending_bytes -= entry.size
+            if (entry.item["ts"] || oldest) < oldest
+              count_drop("stale")
+              next
+            end
+            if entry.array == "slow"
+              if slow >= MAX_SLOW_PER_REQUEST
+                skipped << entry
+                next
+              end
+              slow += 1
+            end
+            batch << entry
+          end
+          skipped.reverse_each do |entry|
+            @pending.unshift(entry)
+            @pending_bytes += entry.size
+          end
+          [batch, heartbeat]
+        end
+      end
+
+      def requeue(batch, heartbeat)
+        @lock.synchronize do
+          batch.reverse_each do |entry|
+            @pending.unshift(entry)
+            @pending_bytes += entry.size
+          end
+          @heartbeat_due = true if heartbeat
+          enforce_bound
+        end
+      end
+
+      # -- sending --------------------------------------------------------------------------------
+
+      def wake
+        @wake_lock.synchronize do
+          @woken = true
+          @wake_cond.signal
+        end
+      end
+
+      def wait_for_wake(seconds)
+        @wake_lock.synchronize do
+          @wake_cond.wait(@wake_lock, seconds) unless @woken
+          woken = @woken
+          @woken = false
+          woken
+        end
+      end
+
+      def start_thread
+        @thread = Thread.new { run }
+        @thread.name = "sillydev-monitoring" if @thread.respond_to?(:name=)
+        @thread.report_on_exception = false
+      end
+
+      def run
+        last_collect = now_mono
+        until @stop
+          woken = wait_for_wake(@flush_interval)
+          break if @stop
+
+          begin
+            now = now_mono
+            if !woken || now - last_collect >= @flush_interval
+              last_collect = now
+              collect
+            end
+            move_ready(false)
+            send_until(now_mono + [REQUEST_TIMEOUT * 2, @flush_interval].max)
+          rescue StandardError => e
+            log_once("thread", :warn, "background flush failed: #{e.class}: #{e.message}")
+          end
+          break if @stopped
+        end
+      end
+
+      def flush_until(deadline)
+        move_ready(true)
+        send_until(deadline)
+      end
+
+      def send_until(deadline)
+        locked = false
+        until (locked = @send_lock.try_lock)
+          return false if now_mono >= deadline
+
+          sleep 0.01
+        end
+        loop do
+          return false if @stopped
+
+          now = now_mono
+          return false if now < @next_attempt || now >= deadline
+
+          batch, heartbeat = take_batch
+          return true if batch.empty? && !heartbeat
+          return false unless deliver(batch, heartbeat, deadline)
+        end
+      ensure
+        @send_lock.unlock if locked
+      end
+
+      def payload(batch, heartbeat)
+        resource = { "service" => @service, "environment" => @environment, "instance" => @instance,
+                     "runtime" => Client.runtime_name }
+        resource["release"] = @release if @release
+        body = { "sdk" => { "name" => "sillydev-ruby", "version" => VERSION }, "resource" => resource,
+                 "heartbeat" => heartbeat ? true : false, "metrics" => [], "errors" => [], "events" => [], "slow" => [] }
+        batch.each { |entry| body[entry.array] << entry.item }
+        body
+      end
+
+      # Sends one batch. True to carry on with the next one; false to stop for now.
+      def deliver(batch, heartbeat, deadline)
+        raw = JSON.generate(payload(batch, heartbeat))
+        content = raw
+        gzipped = false
+        if raw.bytesize >= GZIP_MIN_BYTES
+          packed = gzip(raw)
+          if packed.bytesize < raw.bytesize
+            content = packed
+            gzipped = true
+          end
+        end
+        if raw.bytesize > MAX_RAW_BYTES || content.bytesize > MAX_SENT_BYTES
+          return split(batch, heartbeat, deadline, "batch over the size limit")
+        end
+
+        debug_log("sending #{content.bytesize} bytes: #{raw[0, 8000]}")
+        headers = { "Authorization" => "Bearer #{@key}", "Content-Type" => "application/json",
+                    "User-Agent" => "sillydev-ruby/#{VERSION} monitoring" }
+        headers["Content-Encoding"] = "gzip" if gzipped
+        timeout = (deadline - now_mono).clamp(0.1, REQUEST_TIMEOUT)
+        begin
+          status, response_headers, response_body = post("#{@endpoint}/v1/batch", content, headers, timeout)
+        rescue StandardError => e
+          requeue(batch, heartbeat)
+          back_off("could not reach #{@endpoint}: #{e.class}")
+          return false
+        end
+
+        if status.between?(200, 299)
+          @failures = 0
+          debug_log("#{status} #{response_body.to_s[0, 2000]}")
+          return true
+        end
+        if [401, 403].include?(status)
+          @stopped = true
+          @lock.synchronize do
+            @pending.clear
+            @pending_bytes = 0
+            @aggregator.clear
+            @slow = []
+          end
+          log_once("auth", :warn, "the ingest answered #{status} (#{error_text(response_body)}); monitoring has " \
+                                  "stopped. Check SILLYDEV_MONITORING_KEY.")
+          wake
+          return false
+        end
+        return split(batch, heartbeat, deadline, "the ingest answered 413") if status == 413
+
+        if status == 429
+          requeue(batch, heartbeat)
+          retry_after = finite(response_headers["retry-after"])
+          @next_attempt = now_mono + (retry_after && retry_after >= 0 ? retry_after : backoff_delay)
+          log_once("429", :info, "rate limited by the ingest, waiting")
+          return false
+        end
+        if status >= 500
+          requeue(batch, heartbeat)
+          back_off("the ingest answered #{status}")
+          return false
+        end
+        count_drop("http_#{status}", batch.size)
+        log_once("http:#{status}", :warn, "the ingest refused a batch (#{status}: #{error_text(response_body)}); it was dropped")
+        true
+      end
+
+      def post(url, body, headers, timeout)
+        return @transport.call(url, body, headers, timeout) if @transport
+
+        uri = URI(url)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = timeout
+        http.read_timeout = timeout
+        http.write_timeout = timeout
+        request = Net::HTTP::Post.new(uri)
+        headers.each { |k, v| request[k] = v }
+        request.body = body
+        response = http.start { |h| h.request(request) }
+        response_headers = {}
+        response.each_header { |k, v| response_headers[k.downcase] = v }
+        [response.code.to_i, response_headers, response.body]
+      end
+
+      def gzip(text)
+        io = StringIO.new(+"".b)
+        writer = Zlib::GzipWriter.new(io, 6)
+        writer.write(text)
+        writer.close
+        io.string
+      end
+
+      def split(batch, heartbeat, deadline, why)
+        log_once("split:#{why}", :info, "#{why}, splitting it")
+        if batch.size <= 1
+          count_drop("too_large", batch.size)
+          log_once("too_large", :warn, "dropped an item too large to send")
+          return true
+        end
+        half = batch.size / 2
+        first = batch[0, half]
+        second = batch[half..]
+        unless deliver(first, heartbeat, deadline)
+          requeue(second, false)
+          return false
+        end
+        deliver(second, false, deadline)
+      end
+
+      def backoff_delay
+        delay = [BACKOFF_CAP, BACKOFF_BASE * (2**[@failures, 20].min)].min
+        (delay / 2) + (@random.call * delay / 2)
+      end
+
+      def back_off(why)
+        @failures += 1
+        delay = backoff_delay
+        @next_attempt = now_mono + delay
+        log_once("backoff:#{why}", :info, format("%<why>s; retrying in %<delay>.1fs", why: why, delay: delay))
+      end
+
+      def error_text(body)
+        parsed = JSON.parse(body.to_s)
+        (parsed["code"] || parsed["error"]).to_s[0, 200]
+      rescue StandardError
+        ""
+      end
+
+      def log_once(key, level, message)
+        if @logged.key?(key)
+          debug_log(message)
+          return
+        end
+        @logged[key] = true if @logged.size < 1000
+        log(level, message)
+      end
+
+      def debug_log(message)
+        log(:debug, message) if @debug
+      end
+
+      def log(level, message)
+        text = "sillydev.monitoring: #{message}"
+        if @logger
+          @logger.public_send(level, text)
+        elsif level == :warn || @debug
+          warn(text)
+        end
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end