end_point_blank 0.2.0

data/lib/end_point_blank/writers/delayed_writer.rb

@@ -0,0 +1,110 @@
+require 'singleton'
+
+module EndPointBlank
+  module Writers
+    # Drains a background send queue with a fixed pool of worker threads.
+    #
+    # The queue is bounded (see MAX_QUEUE_SIZE) so that during an intake
+    # outage - when the worker threads can't drain as fast as the host
+    # application enqueues payloads - memory usage stays capped instead of
+    # growing without bound toward an OOM. When the queue is full, the
+    # *oldest* item is dropped to make room for the newest one, and a
+    # warning is logged, throttled so a sustained outage does not itself
+    # become a logging flood.
+    module DelayedWriter
+      MAX_QUEUE_SIZE = 1000
+      WARN_THROTTLE_SECONDS = 30
+
+      def direct_writer
+        @direct_writer ||= DirectWriter.new(url)
+      end
+
+      def queue
+        @queue ||= Queue.new
+      end
+
+      def start_threads
+        @threads = []
+
+        2.times do
+          @threads << Thread.new do
+            loop do
+              payload = queue.pop
+              payloads = [payload]
+              while (payload = pop_additional) do
+                payloads << payload
+              end
+
+              payloads.compact!
+              while payloads.any? do
+                list = payloads[0..5]
+                response = direct_writer.write(list)
+                if response.status < 299
+                  on_success(response) if respond_to?(:on_success)
+                else
+                  on_failure(response) if respond_to?(:on_failure)
+                end
+                payloads = payloads - list
+              end
+            end
+          end
+        end
+      end
+
+      def pop_additional
+        queue.pop(true)
+      rescue ThreadError
+        nil
+      end
+
+      def enqueue(list)
+        items = list.is_a?(Array) ? list : [list]
+        items.each { |payload| enqueue_one(payload) }
+      end
+
+      # Logs a drop warning. Overridable/stubbable in tests; routed through
+      # the pluggable EndPointBlank.logger seam (this lib is fire-and-forget
+      # and must never raise).
+      def log_warning(message)
+        EndPointBlank.logger.warn(message)
+      end
+
+      private
+
+      def enqueue_mutex
+        @enqueue_mutex ||= Mutex.new
+      end
+
+      def enqueue_one(payload)
+        enqueue_mutex.synchronize do
+          drop_oldest_and_note if queue.size >= MAX_QUEUE_SIZE
+          queue << payload
+        end
+      end
+
+      def drop_oldest_and_note
+        pop_additional
+        note_drop
+      end
+
+      def note_drop
+        @dropped_since_last_warning = (@dropped_since_last_warning || 0) + 1
+        now = Time.now
+        return if @last_drop_warning_at && (now - @last_drop_warning_at) < WARN_THROTTLE_SECONDS
+
+        warn_dropped_items(now)
+      end
+
+      def warn_dropped_items(now)
+        dropped = @dropped_since_last_warning
+        @dropped_since_last_warning = 0
+        @last_drop_warning_at = now
+
+        log_warning(
+          "[EndPointBlank] send queue full (max #{MAX_QUEUE_SIZE}); " \
+          "dropped #{dropped} oldest item(s) since last warning"
+        )
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/direct_writer.rb

@@ -0,0 +1,18 @@
+require_relative '../commands/http'
+
+module EndPointBlank
+  module Writers
+    class DirectWriter
+      attr_accessor :url
+
+      def initialize(url)
+        @url = url
+      end
+
+      def write(list)
+        auth = EndPointBlank::Authorization.header
+        EndPointBlank::Commands::Http.post(@url, auth, { payload: list })
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/exception_writer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module EndPointBlank
+  module Writers
+    class ExceptionWriter
+      include Singleton
+      include DelayedWriter
+      include Shared
+
+      attr_reader :url
+
+      def initialize
+        @url = EndPointBlank::Configuration.instance.errors_url
+        start_threads
+      end
+
+      def self.write(exception)
+        instance.write(exception)
+      end
+
+      def payload(exception)
+        env = ::EndPointBlank::Rack::EnvStore.get
+        request = ::Rack::Request.new(env)
+        version = Commands::VersionFinder.new.find(request)
+        {
+          app_name: app_name,
+          uuid: request_uuid(env),
+          message: exception.message,
+          stacktrace: exception.backtrace,
+          sent_at: Time.now.utc.iso8601(3),
+          source_application_environment_id: source_application_environment_id
+        }
+      end
+
+      def write(exception)
+        p = payload(exception)
+        rack_req = ::EndPointBlank::Rack::EnvStore.request
+        if rack_req
+          p = p.merge(
+            stamped_path: rack_req.path,
+            stamped_http_method: rack_req.request_method
+          )
+        end
+        enqueue(apply_masking(p, :error))
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/log_writer.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module EndPointBlank
+  module Writers
+    class LogWriter
+      include Singleton
+      include DelayedWriter
+      include Shared
+
+      attr_reader :url
+
+      def initialize
+        @url = EndPointBlank::Configuration.instance.logs_url
+        start_threads
+      end
+
+      def self.info(message, data = {})
+        instance.write(message, :info, data)
+      end
+
+      def self.warn(message, data = {})
+        instance.write(message, :warn, data)
+      end
+
+      def self.error(message, data = {})
+        instance.write(message, :error, data)
+      end
+
+      def self.fatal(message, data = {})
+        instance.write(message, :fatal, data)
+      end
+
+      def self.write(message, level, data = {})
+        instance.write(message, level, data)
+      end
+
+      def payload(message: message, level: level, data: data)
+        env = ::EndPointBlank::Rack::EnvStore.get
+        uuid = request_uuid(env)
+
+        {
+          message: message,
+          log_level: level,
+          sent_at: Time.now.utc.iso8601(3),
+          app_name: app_name,
+          uuid: uuid,
+          data: data,
+          source_application_environment_id: source_application_environment_id
+        }
+      end
+
+
+      def write(message, level, data = {})
+        json = payload(message: message, level: level, data: data)
+        rack_req = ::EndPointBlank::Rack::EnvStore.request
+        if rack_req
+          json = json.merge(
+            stamped_path: rack_req.path,
+            stamped_http_method: rack_req.request_method
+          )
+        end
+        puts "Writing log: #{json}"
+        enqueue(json)
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/request_writer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module EndPointBlank
+  module Writers
+    class RequestWriter
+      include Singleton
+      include DelayedWriter
+      include Shared
+
+      attr_reader :url
+
+      def initialize
+        @url = EndPointBlank::Configuration.instance.requests_url
+        start_threads
+      end
+
+      def self.write(data = {})
+        instance.write()
+      end
+
+      def payload
+        env = ::EndPointBlank::Rack::EnvStore.get
+        request = ::Rack::Request.new(env)
+        version = Commands::VersionFinder.new.find(request)
+        headers = ::EndPointBlank::Rack::Headers.extract
+
+        payload = {
+          app_name: EndPointBlank::Configuration.instance.app_name,
+          env: SessionConfiguration.env_name,
+          uuid: request_uuid(env),
+          host: request.host,
+          status: request_status(request),
+          headers: headers,
+          path: request.path,
+          http_method: request.request_method,
+          endpoint_version: version,
+          request: request_body(request),
+          sent_at: Time.now.utc.iso8601(3)
+        }
+      end
+
+      def write()
+        enqueue(apply_masking(payload, :request))
+      end
+
+      private
+
+      def request_status(request)
+        request.respond_to?(:status) ? request.status : nil
+      end
+
+      def request_body(request)
+        body = request.body&.read
+        request.body.rewind if request.body&.respond_to?(:rewind)
+        body
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/response_writer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "singleton"
+
+module EndPointBlank
+  module Writers
+    class ResponseWriter
+      include Singleton
+      include DelayedWriter
+      include Shared
+
+      attr_reader :url
+
+      def initialize
+        @url = EndPointBlank::Configuration.instance.responses_url
+        start_threads
+      end
+
+      def self.write(status:, headers: {}, body: nil, data: {})
+        instance.write(status:, headers:, body:, data:)
+      end
+
+      def payload(status:, headers:, body:, data: {})
+        request = ::EndPointBlank::Rack::EnvStore.request
+        env = ::EndPointBlank::Rack::EnvStore.get
+        headers = ::EndPointBlank::Rack::Headers.extract
+        version = request ? Commands::VersionFinder.new.find(request) : nil
+        route = request ? Commands::RoutePatternFinder.find(request) : nil
+
+        {
+            app_name: app_name,
+            env: env_name,
+            uuid: request_uuid(env),
+            status: status,
+            headers: headers,
+            body: truncate(normalize_body(body)),
+            sent_at: Time.now.utc.iso8601(3),
+            route: route,
+            data: data,
+            source_application_environment_id: source_application_environment_id
+        }
+      end
+
+      def write(status:, headers: {}, body: nil, data: {})
+        enqueue(apply_masking(payload(status:, headers:, body:, data:), :response))
+      end
+
+      private
+      def truncate(body)
+        return body if body.nil? || body.length <= 1024
+
+        body[0...1024] + "..."
+      end
+
+      def normalize_body(body)
+        return nil if body.nil?
+
+        if body.respond_to?(:body) && !body.is_a?(Array)
+          body.body
+        elsif body.respond_to?(:each)
+          result = +""
+          body.each { |chunk| result << chunk }
+          result
+        else
+          body.to_s
+        end
+      end
+    end
+  end
+end

data/lib/end_point_blank/writers/shared.rb

@@ -0,0 +1,39 @@
+module EndPointBlank
+  module Writers
+    module Shared
+      attr_accessor :url
+
+      def configuration
+        Configuration.instance
+      end
+
+      def app_name
+        configuration.app_name
+      end
+
+      def env_name
+        SessionConfiguration.env_name
+      end
+
+      def env
+        ::EndPointBlank::Rack::EnvStore.get
+      end
+
+      def source_application_environment_id
+        ::EndPointBlank::Rack::EnvStore.source_application_environment_id
+      end
+
+      # Rails' ActionDispatch::Request#uuid simply reads this same Rack env
+      # key, so this is behavior-preserving when running under Rails, and
+      # gracefully returns nil when it isn't (plain Rack / Sinatra), instead
+      # of requiring actionpack's ActionDispatch::Request to be loaded.
+      def request_uuid(env)
+        env && env["action_dispatch.request_id"]
+      end
+
+      def apply_masking(payload, record_type)
+        EndPointBlank::Masking.apply(payload, record_type, configuration.masking_rules, configuration.mask_hook)
+      end
+    end
+  end
+end

data/lib/end_point_blank/xml_truncator.rb

@@ -0,0 +1,89 @@
+require "rexml/document"
+require "rexml/formatters/default"
+
+class XmlTruncator
+  MAX_BYTES = 10000
+  MAX_DEPTH = 6
+  MAX_CHILDREN = 20
+  MAX_ATTRIBUTES = 20
+  MAX_TEXT = 200
+  TEXT_SUFFIX = "..."
+
+  def self.truncate(xml, limit: MAX_BYTES)
+    input = xml.to_s
+    return "" if input.empty?
+    return input if input.bytesize <= limit
+
+    document = parse_xml(input)
+    return StringTruncator.truncate(input, limit:, suffix: "<truncated/>") unless document&.root
+
+    pruned_root = prune_element(document.root, depth: 0)
+    output = render_element(pruned_root)
+    return output if output.bytesize <= limit
+
+    compact = compact_fallback(document.root.expanded_name)
+    return compact if compact.bytesize <= limit
+
+    "<truncated/>"
+  end
+
+  def self.parse_xml(input)
+    REXML::Document.new(input)
+  rescue REXML::ParseException
+    nil
+  end
+
+  def self.prune_element(element, depth:)
+    pruned = REXML::Element.new(element.expanded_name)
+
+    element.attributes.each_with_index do |(name, value), idx|
+      break if idx >= MAX_ATTRIBUTES
+      pruned.add_attribute(name, truncate_text(value, max: 100))
+    end
+
+    if depth >= MAX_DEPTH
+      pruned.add_element("truncated")
+      return pruned
+    end
+
+    element_count = 0
+
+    element.children.each do |child|
+      case child
+      when REXML::Element
+        if element_count >= MAX_CHILDREN
+          pruned.add_element("truncated")
+          break
+        end
+
+        pruned.add_element(prune_element(child, depth: depth + 1))
+        element_count += 1
+      when REXML::CData
+        pruned.add(REXML::CData.new(truncate_text(child.value)))
+      when REXML::Text
+        text = truncate_text(child.value)
+        pruned.add_text(text) unless text.empty?
+      end
+    end
+
+    pruned
+  end
+
+  def self.render_element(element)
+    formatter = REXML::Formatters::Default.new
+    output = +""
+    formatter.write(element, output)
+    output
+  end
+
+  def self.compact_fallback(root_name)
+    "<#{root_name}><truncated/></#{root_name}>"
+  end
+
+  def self.truncate_text(text, max: MAX_TEXT)
+    value = text.to_s
+    return value if value.bytesize <= max
+
+    value.byteslice(0, max - TEXT_SUFFIX.bytesize) + TEXT_SUFFIX
+  end
+end

data/lib/end_point_blank.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "logger"
+require "rack"
+
+require_relative "end_point_blank/access_tokens"
+require_relative "end_point_blank/authorization"
+require_relative "end_point_blank/version"
+require_relative "end_point_blank/configuration"
+require_relative "end_point_blank/session_configuration"
+require_relative "end_point_blank/log_entry"
+require_relative "end_point_blank/string_truncator"
+require_relative "end_point_blank/fast_json_truncator"
+require_relative "end_point_blank/xml_truncator"
+require_relative "end_point_blank/masking"
+require_relative "end_point_blank/writers/shared"
+require_relative "end_point_blank/writers/delayed_writer"
+require_relative "end_point_blank/writers/direct_writer"
+require_relative "end_point_blank/writers/log_writer"
+require_relative "end_point_blank/writers/exception_writer"
+require_relative "end_point_blank/writers/request_writer"
+require_relative "end_point_blank/writers/response_writer"
+require_relative "end_point_blank/commands/generate_access_token"
+require_relative "end_point_blank/commands/authentication_cache"
+require_relative "end_point_blank/commands/basic_authenticate"
+require_relative "end_point_blank/commands/bearer_generate"
+require_relative "end_point_blank/commands/endpoint_authorize"
+require_relative "end_point_blank/commands/route_pattern_finder"
+require_relative "end_point_blank/commands/endpoint_update"
+require_relative "end_point_blank/commands/version_finder"
+require_relative "end_point_blank/middleware/rack/report_interaction"
+require_relative "end_point_blank/rack/env_store"
+require_relative "end_point_blank/rack/headers"
+require_relative "end_point_blank/unauthorized_error"
+if defined?(::Rails)
+  require_relative "end_point_blank/rails/authenticated"
+  require_relative "end_point_blank/rails/authorized"
+  require_relative "end_point_blank/rails/versioned"
+  require_relative "end_point_blank/rails/railtie"
+end
+
+module EndPointBlank
+  class Error < StandardError; end
+
+  # Your code goes here...
+  def self.configure(&block)
+    yield Configuration.instance
+  end
+
+  def self.logger
+    Configuration.instance.logger || (@default_logger ||= ::Logger.new($stdout, level: ::Logger::INFO))
+  end
+
+  def self.logger=(logger)
+    Configuration.instance.logger = logger
+  end
+end

data/sig/end_point_blank_rack.rbs

@@ -0,0 +1,4 @@
+module EndPointBlankRack
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/test.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+# Run the test suite for end_point_blank (Ruby gem).
+set -euo pipefail
+cd "$(dirname "$0")"
+exec bundle exec rspec "$@"