invoca-metrics 0.0.2 → 1.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c61c3ea9d85deef7f1abf581660aad82dee010267632bd522c523a6a52a5b67c
-  data.tar.gz: 39b26c15860fa3ec223dabb0b0a6a18d7019e9e6e7f1585934d948c0d0453e2c
+  metadata.gz: 90b8517fa182fbf84e8067d0f4727aac492ea493804f5913cbd0495a3c1a1a88
+  data.tar.gz: f694ecc164dc329381989fbce06d5ea91fb416707d54ca9a571eeda286c3ab52
 SHA512:
-  metadata.gz: '009179dfd5fe4f2e69310d7f0aa6dd210b80fd1d8749c55e736162f3c150d78653e139bef8553725a0136667b2444c783a326a76c12ac39bfb2d602fcec60d71'
-  data.tar.gz: c19157d85d447dcf6496b2751cd0565f7a21bc1559bfb1a59009ff524d20193e28e4a95b197ae97439cf45fd342ed13f8d446c6704fbd025d926611704b1d430
+  metadata.gz: db2b4c7f76eb190ce6e2ff8231f5eb1f39103fde83e314d4eb40d18c1b01a73820e0b253be0969bdcbf179ea8cbdb4466845182c952ee1a159f736486bd041ed
+  data.tar.gz: b48fd3176d18a7957c6cf98e88513044c275195f8ab93036192f0599b9c338d84c615b94ad8465f468318f4513ecd42fca381a2ef180d17541c09b352c689a2e

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Cary Penniman
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,161 @@
+# Invoca::Metrics
+
+Publish metrics for observability!
+
+## Dependencies
+* Ruby >= 2.6
+* ActiveSupport >= 4.2, < 7
+
+## Usage
+
+Mix in the `Invoca::Metrics::Source` module:
+```ruby
+class MyClass
+  include Invoca::Metrics::Source
+  ...
+end
+```
+
+Then declare the metrics that can be published by the class using a `declare_metrics` DSL block.
+Each entry in the block starts with the metric type as the DSL method:
+
+| Metric Type | Value                                               | Metric Instance Methods                                                                                                                                                                                                                                            |
+|:-----------:|:----------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|  `counter`  | monotonically incrementing floating point           | `.increment(value = 1, **labels)`                                                                                                                                                                                                                                  |
+|   `gauge`   | arbitrary floating point                            | `.set(value, **labels)`                                                                                                                                                                                                                                            |
+| `histogram` | histogram counters with configurable bucket cutoffs | `.add(value, **labels)`<br/><br/>`.time(return_timing: false, **labels, &block)`<br/>yields to block and `add()`s the milliseconds spent to the histogram; returns either `block_value` or `[milliseconds, block_value]` depending on the `return_timing` argument |
+
+After the metric type, the first argument is the metric name, given as a symbol. This is globally unique.
+Note that other classes in the process are allowed to declare the same metric, as long as the type and labels are the same.
+
+Each metric may have any number of labels declared in the optional `labels:` hash, with default values in case that label is not passed when the metric is published.
+
+Note: An implicit label of `service: <service_name>` is always added. The `<service_name>` is a configured value.
+
+Histograms have a required argument, `buckets: <array>`, where the array is the histogram bucket cutoffs.
+
+```ruby
+class MyClass
+  include Invoca::Metrics::Source
+
+  declare_metrics do
+    counter   :request_total, labels: { status: '200' }
+    gauge     :available_memory
+    histogram :processing_time, buckets: [30.seconds, 1.hour, 2.hours, 4.hours, 5.hours, 6.hours, 10.hours]
+  end
+
+  ...
+end
+```
+
+After declaring the metrics, you may use `prometheus_metrics.<metric_name>` (in either the class or instance) to publish the metric.
+```ruby
+prometheus_metrics.request_total.increment       # increment the counter by 1
+
+prometheus_metrics.request_total.increment(5)    # increment the counter by 5
+
+prometheus_metrics.available_memory.set(10.25)   # set the gauge to 10.25
+
+prometheus_metrics.processing_time.add(100)      # Increment the bucket closest to the value 100
+prometheus_metrics.processing_time.time { work } # measure the elapsed milliseconds to run the given block and increment the bucket closest to that time period in milliseconds
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'invoca-metrics'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install invoca-metrics
+
+## Setup
+
+### Default Setup
+
+Add the following code to your application...
+
+    require 'invoca/metrics'
+
+    Invoca::Metrics.service_name    = "tts-service"
+    Invoca::Metrics.server_name     = Socket.gethostname
+    Invoca::Metrics.cluster_name    = "production"
+    Invoca::Metrics.sub_server_name = "worker_process_1"
+    Invoca::Metrics.statsd_host     = "255.0.0.123"
+    Invoca::Metrics.statsd_port     = 200
+
+    Invoca::Metrics::Client.logger = logger
+    Invoca::Metrics::Client.log_send_failures = ['staging', 'production'].include(environment)
+
+Note:  the `service_name` setting is required; all others are optional.
+
+### Multi Configuration
+
+To set multiple configurations, supply a `config` hash with additional settings.
+
+    Invoca::Metrics.config = {
+      deployment_group: {
+        server_name: "deployment"
+        statsd_host: "255.0.0.456",
+        statsd_port: 300
+      },
+      region: {
+        server_name:     "region_name"
+        statsd_host:     "255.0.0.789",
+        statsd_port:     400,
+        sub_server_name: nil
+        cluster_name:    nil
+      }
+    }
+
+The settings (`[service_name, server_name, cluster_name, sub_server_name, statsd_host, statsd_port]`) directly set on `Invoca::Metrics` will be the default values supplied if the individual configuration does not supply the option.
+It's highly suggested that each configuration has its own `statsd_host` and `statsd_port` along with unique naming since writing the same metric from one statsd node could be overwritten by the same metric from a separate node.
+
+In order to set a configuration as the default, set the configuration key as `default_config_key`.
+
+    Invoca::Metrics.default_config_key = :deployment_group
+
+Any keys missing from the `default_config_key` config will by default have the values from the keys set directly on `Invoca::Metrics`.
+
+The full set of default values for the above example would then be
+
+    service_name:    "my_event_worker"
+    cluster_name:    "production"
+    sub_server_name: "worker_process_1"
+    server_name:     "deployment"
+    statsd_host:     "255.0.0.456",
+    statsd_port:     300
+
+### Prometheus Metric Exporting Setup
+There are 2 ways to set up metrics for being scraped by Prometheus: `:direct` and `:push`).
+`:direct` will expose a web `server directly from the running process in order for metrics to be scraped while `:push` will push metrics over to an aggregate gateway which in turn implements a web server.
+
+#### `:direct` Configuration Example
+```ruby
+Invoca::Metrics::Prometheus.configure do |config|
+  config.mode = :direct
+  config.port = 9394 # Port to expose metrics from
+end
+```
+
+#### `:push` Configuration
+```ruby
+Invoca::Metrics::Prometheus.configure do |config|
+  config.mode = :push
+  config.host = '127.0.0.1' # IP address to push metrics to
+  config.port = 9394 # Port to push metrics to
+end
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/invoca-metrics.gemspec

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'invoca/metrics/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "invoca-metrics"
+  spec.version       = Invoca::Metrics::VERSION
+  spec.authors       = ["Invoca development"]
+  spec.email         = ["development@invoca.com"]
+  spec.description   = 'Invoca metrics reporting library'
+  spec.summary       = 'Invoca metrics reporting library'
+  spec.homepage      = "https://github.com/Invoca/invoca-metrics"
+  spec.license       = "MIT"
+
+  spec.metadata['allowed_push_host'] = "https://rubygems.org"
+
+  spec.files         = [
+      *Dir.glob('lib/**/*.rb'),
+      'README.md',
+      'LICENSE.txt',
+      'invoca-metrics.gemspec',
+  ]
+
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport",       ">= 4.2", "< 7"
+  spec.add_dependency "statsd-ruby",         "~> 1.2.1"
+  spec.add_dependency "prometheus_exporter", "~> 2.0"
+  spec.add_dependency "contextual_logger",   "~> 1.0"
+end

data/lib/invoca/metrics/batch.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/module/delegation'
+
+module Invoca
+  module Metrics
+    class Batch < Client
+      delegate :batch_size, :batch_size=, to: :statsd_client
+
+      # @param [Invoca::Metrics::Client] client requires a configured Client instance
+      # @param [Statsd::Batch] statsd_batch requires a configured Batch instance
+      def initialize(client, statsd_batch)
+        super(
+          hostname: client.hostname,
+          port: client.port,
+          cluster_name: client.cluster_name,
+          service_name: client.service_name,
+          server_label: client.server_label,
+          sub_server_name: client.sub_server_name,
+          namespace: client.namespace
+        )
+        @statsd_client = statsd_batch
+      end
+
+      # @yields [Batch] yields itself
+      #
+      # A convenience method to ensure that data is not lost in the event of an
+      # exception being thrown. Batches will be transmitted on the parent socket
+      # as soon as the batch is full, and when the block finishes.
+      def ensure_send
+        yield self
+      ensure
+        statsd_client.flush
+      end
+    end
+  end
+end

data/lib/invoca/metrics/client.rb

@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/module/delegation'
+
+module Invoca
+  module Metrics
+    class Client
+      STATSD_DEFAULT_HOSTNAME = "127.0.0.1"
+      STATSD_DEFAULT_PORT = 8125
+      STATSD_METRICS_SEPARATOR = '.'
+
+      class << self
+        delegate :logger, :logger=, :log_send_failures, :log_send_failures=, to: StatsdClient
+
+        # Default values are required for backwards compatibility
+        def metrics(statsd_host:     Invoca::Metrics.default_client_config[:statsd_host],
+                    statsd_port:     Invoca::Metrics.default_client_config[:statsd_port],
+                    cluster_name:    Invoca::Metrics.default_client_config[:cluster_name],
+                    service_name:    Invoca::Metrics.default_client_config[:service_name],
+                    server_name:     Invoca::Metrics.default_client_config[:server_name],
+                    sub_server_name: Invoca::Metrics.default_client_config[:sub_server_name],
+                    namespace:       nil)
+          config = {
+            hostname:        statsd_host || STATSD_DEFAULT_HOSTNAME,
+            port:            statsd_port || STATSD_DEFAULT_PORT,
+            cluster_name:    cluster_name,
+            service_name:    service_name,
+            server_label:    server_name,
+            sub_server_name: sub_server_name,
+            namespace:       namespace
+          }.freeze
+
+          client_cache[config] ||= new(**config)
+        end
+
+        def reset_cache
+          @client_cache = {}
+        end
+
+        private
+
+        def client_cache
+          @client_cache ||= {}
+        end
+      end
+
+      attr_reader :hostname, :port, :server_label, :sub_server_name, :cluster_name, :service_name, :gauge_cache
+      delegate :batch_size, :namespace, :timing, :time, to: :statsd_client
+
+      def initialize(hostname:, port:, cluster_name: nil, service_name: nil, server_label: nil, sub_server_name: nil, namespace: nil)
+        @hostname        = hostname
+        @port            = port
+        @cluster_name    = cluster_name
+        @service_name    = service_name
+        @server_label    = server_label
+        @sub_server_name = sub_server_name
+
+        @statsd_client = StatsdClient.new(@hostname, @port)
+        @statsd_client.namespace = namespace || [@cluster_name, @service_name].compact.join(STATSD_METRICS_SEPARATOR).presence
+
+        @gauge_cache = GaugeCache.register(gauge_cache_key, @statsd_client)
+      end
+
+      def gauge_cache_key
+        [
+          hostname,
+          port,
+          cluster_name,
+          service_name,
+          namespace,
+          server_name,
+          sub_server_name
+        ].freeze
+      end
+
+      def server_name # For backwards compatibility
+        server_label
+      end
+
+      # This will store the gauge value passed in so that it is reported every GAUGE_REPORT_INTERVAL
+      # seconds and post the gauge at the same time to avoid delay in gauges being
+      def gauge(name, value)
+        log_usage(name, :gauge)
+        if (args = normalized_metric_name_and_value(name, value, "gauge"))
+          gauge_cache.set(*args)
+          statsd_client.gauge(*args)
+        end
+      end
+
+      def count(name, value = 1)
+        log_usage(name, :counter)
+        if (args = normalized_metric_name_and_value(name, value, "counter"))
+          statsd_client.count(*args)
+        end
+      end
+
+      alias counter count
+
+      def increment(name)
+        count(name, 1)
+      end
+
+      def decrement(name)
+        count(name, -1)
+      end
+
+      def set(name, value)
+        log_usage(name, :counter)
+        if (args = normalized_metric_name_and_value(name, value, nil))
+          statsd_client.set(*args)
+        end
+      end
+
+      def timer(name, milliseconds = nil, return_timing: false, &block)
+        name.present? or raise ArgumentError, "Must specify a metric name."
+        (!milliseconds.nil? ^ block_given?) or raise ArgumentError, "Must pass exactly one of milliseconds or block."
+        name_and_type = [name, "timer", server_label].join(STATSD_METRICS_SEPARATOR)
+        log_usage(name, :timer)
+
+        if milliseconds.nil?
+          result, block_time = time(name_and_type, &block)
+          return_timing ? [result, block_time] : result
+        else
+          timing(name_and_type, milliseconds)
+        end
+      end
+
+      def batch(&block)
+        statsd_client.batch do |batch|
+          Metrics::Batch.new(self, batch).ensure_send(&block)
+        end
+      end
+
+      # TODO: - implement transmit method
+      def transmit(message, extra_data = {})
+        # TODO: - we need to wire up exception data to a monitoring service
+      end
+
+      private
+
+      attr_reader :statsd_client
+
+      def normalized_metric_name_and_value(name, value, stat_type)
+        name.present? or raise ArgumentError, "Must specify a metric name."
+        extended_name = [name, stat_type, @server_label, @sub_server_name].compact.join(STATSD_METRICS_SEPARATOR)
+        if value
+          [extended_name, value]
+        else
+          [extended_name]
+        end
+      end
+
+      def log_usage(metric_name, metric_type)
+        if Invoca::Metrics.graphite_usage_logging_enabled
+          call_stack = caller_locations
+          unless call_stack.any? { |l| /invoca\/metrics\/prometheus/.match?(l.to_s) }
+            self.class.logger&.info(
+              "Deprecated usage of grapihite metrics",
+              invoca_metrics: {
+                metric_name:   metric_name,
+                metric_type:   metric_type,
+                metric_source: call_stack.find { |l| !/invoca\/metrics/.match?(l.to_s) }.to_s
+              }
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/direct_metric.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Invoca
+  module Metrics
+    # Directly reports metrics without sending through graphite.  Does not add process information to metric names.
+    class DirectMetric
+      attr_reader :name, :value, :tick
+
+      def initialize(name, value, tick = nil)
+        @name = name
+        @value = value
+        @tick = tick || self.class.rounded_tick
+      end
+
+      def to_s
+        "#{name} #{value} #{tick}"
+      end
+
+      PERIOD = 60
+      DEFAULT_PORT = 2003
+      DEFAULT_HOST = '127.0.0.1'
+
+      class << self
+        def report(metrics)
+          metrics_output = Array(metrics).map { |metric| "#{metric}\n" }.join
+
+          send_to_metric_host(metrics_output)
+        end
+
+        def generate_distribution(metric_prefix, metric_values, tick = nil)
+          fixed_tick = tick || rounded_tick
+          sorted_values = metric_values.sort
+          count = sorted_values.count
+
+          if count == 0
+            [
+              new("#{metric_prefix}.count",    count,                    fixed_tick)
+            ]
+          else
+            [
+              new("#{metric_prefix}.count",    count,                    fixed_tick),
+              new("#{metric_prefix}.max",      sorted_values[-1],        fixed_tick),
+              new("#{metric_prefix}.min",      sorted_values[0],         fixed_tick),
+              new("#{metric_prefix}.median",   sorted_values[count * 0.5], fixed_tick),
+              new("#{metric_prefix}.upper_90", sorted_values[count * 0.9], fixed_tick)
+            ]
+          end
+        end
+
+        def rounded_tick
+          tick = Time.now.to_i
+          tick - (tick % PERIOD)
+        end
+
+        private
+
+        def send_to_metric_host(message)
+          host = ENV["DIRECT_METRIC_HOST"] || DEFAULT_HOST
+          port = (ENV["DIRECT_METRIC_PORT"] || DEFAULT_PORT).to_i
+
+          TCPSocket.open(host, port) do |tcp|
+            tcp.send(message, 0)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/gauge_cache.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Invoca
+  module Metrics
+    class GaugeCache
+      GAUGE_REPORT_INTERVAL = 60.seconds
+
+      class << self
+        def register(cache_key, statsd_client)
+          registered_gauge_caches[cache_key] ||= new(statsd_client)
+        end
+
+        def reset
+          @registered_gauge_caches = {}
+        end
+
+        private
+
+        def registered_gauge_caches
+          @registered_gauge_caches ||= {}
+        end
+      end
+
+      attr_reader :cache
+
+      def initialize(statsd_client)
+        @statsd_client = statsd_client
+        @cache = {}
+        start_reporting_thread
+      end
+
+      # Atomic method for setting the value for a particular gauge
+      def set(metric, value)
+        @cache[metric] = value
+      end
+
+      # Reports all gauges that have been set in the cache
+      # To avoid "RuntimeError: can't add a new key into hash during iteration" from occurring we are
+      # temporarily duplicating the cache to iterate and send the batch of metrics
+      def report
+        @statsd_client.batch do |statsd_batch|
+          @cache.dup.each do |metric, value|
+            statsd_batch.gauge(metric, value) if value
+          end
+        end
+      end
+
+      def service_environment
+        @service_environment ||= ENV['RAILS_ENV'].presence || ENV['SERVICE_ENV'].presence || 'development'
+      end
+
+      private
+
+      def start_reporting_thread
+        Thread.new do
+          reporting_loop_with_rescue
+        end
+      end
+
+      def reporting_loop_with_rescue
+        reporting_loop
+      rescue Exception => ex
+        Invoca::Metrics::Client.logger.error("GaugeCache#reporting_loop_with_rescue rescued exception:\n#{ex.class}: #{ex.message}")
+      end
+
+      def reporting_loop
+        next_time = Time.now.to_i
+        loop do
+          next_time = (((next_time + GAUGE_REPORT_INTERVAL + 1) / GAUGE_REPORT_INTERVAL) * GAUGE_REPORT_INTERVAL) - 1
+          report
+          if (delay = next_time - Time.now.to_i) > 0
+            sleep(delay)
+          else
+            warn("Window to report gauge may have been missed.") unless service_environment == 'test'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/configuration.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'prometheus_exporter'
+require 'prometheus_exporter/server'
+require 'prometheus_exporter/client'
+
+module Invoca
+  module Metrics
+    module Prometheus
+      class Configuration
+        AVAILABLE_MODES        = [:direct, :push].freeze
+        AVAILABLE_METRIC_TYPES = [:counter, :gauge, :histogram].freeze
+
+        # The mode to use for exporting Prometheus metrics.
+        # Valid modes are:
+        #   - `:direct` (default):  Exports metrics directly using a threaded webserver.
+        #   - `:push`: Exports metrics using a pushgateway.
+        #
+        # @return [Symbol]
+        attr_reader :mode
+
+        # The host to use for the Prometheus server. This setting
+        # is only used when in :push mode. Defaults to `localhost`.
+        #
+        # @return [String]
+        attr_accessor :host
+
+        # The port to use when exporting metrics. When in :direct mode
+        # this is used to specify the port to listen on. When in :push
+        # mode this is used to specify the port to push to. Defaults to 9394.
+        #
+        # @return [Integer]
+        attr_accessor :port
+
+        # The client to use for exporting Prometheus metrics.
+        #
+        # @return [PrometheusExporter::Client]
+        attr_reader :client
+
+        # The webserver that is exposed when in :direct mode.
+        #
+        # @return [PrometheusExporter::Server::WebServer]
+        attr_reader :server
+
+        def initialize
+          @host = 'localhost'
+          @port = 9394
+        end
+
+        def mode=(new_mode)
+          AVAILABLE_MODES.include?(new_mode) or raise ArgumentError, "Invalid mode #{new_mode.inspect}. Allowed modes: #{AVAILABLE_MODES}"
+          @mode = new_mode
+        end
+
+        # Initializes the client to use for exporting Prometheus metrics and
+        # freezes the configuration object so no more changes can be made to it.
+        def freeze
+          validate_configuration!
+
+          case mode
+          when :direct
+            initialize_direct_client
+          when :push
+            initialize_push_client
+          end
+
+          PrometheusExporter::Client.default = @client
+
+          super
+        end
+
+        # Registers a metric against the current Configuration. Once the metric object
+        # is registered, metrics can be reported and exported through the metric object.
+        #
+        # @param type [Symbol] The type of metric to register. Valid types are: :counter, :gauge, :histogram
+        # @param name [String] The name of the metric.
+        # @param help [String] The help text for the metric.
+        # @param opts [Hash] A hash of options to pass to the metric object. This is different depending on the
+        #   type of metrics being registered.
+        #
+        # @return metric [PrometheusExporter::Metric]
+        def register_metric(type, name, help, opts = nil)
+          AVAILABLE_METRIC_TYPES.include?(type) or raise ArgumentError, "Invalid metric type #{type.inspect}. Allowed types: #{AVAILABLE_METRIC_TYPES}"
+          client.register(type, name, help, opts)
+        end
+
+        private
+
+        def initialize_direct_client
+          @server = PrometheusExporter::Server::WebServer.new(bind: '0.0.0.0', port: port)
+          @server.start
+          @client = PrometheusExporter::LocalClient.new(collector: @server.collector)
+        end
+
+        def initialize_push_client
+          @client = PrometheusExporter::Client.new(host: host, port: port)
+        end
+
+        def validate_configuration!
+          mode.present? or raise ArgumentError, "Mode must be provided. Valid modes: #{AVAILABLE_MODES}"
+          port.present? or raise ArgumentError, "Port must be provided when in #{mode.inspect} mode"
+          if mode == :push
+            host.present? or raise ArgumentError, "Host must be provided when in #{mode.inspect} mode"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/declare_metrics/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Invoca
+  module Metrics
+    module Prometheus
+      module DeclareMetrics
+        class Base
+
+          attr_reader :name, :type, :source, :metric, :default_labels, :graphite
+
+          def initialize(name, type, source, metric, labels:, graphite:)
+            @name           = name
+            @type           = type
+            @source         = source
+            @metric         = metric
+            @default_labels = labels
+            @graphite       = graphite
+
+            if labels
+              labels.is_a?(Hash) or raise ArgumentError, "Labels must be type Hash but got #{labels.inspect}"
+            end
+          end
+
+          # TODO: Could default_labels be renamed to labels?
+          def settings
+            { type: type, labels: default_labels, graphite: graphite }
+          end
+
+          # TODO: This is temporary; the invoca-metrics metrics objects
+          # should be made immutable, in which case the entire object
+          # would be replaced rather than resetting the Prometheus metric
+          # object as is done here
+          def reset_metric(new_metric)
+            @metric = new_metric
+          end
+
+          private
+
+          def render_graphite_string(**labels)
+            rendered_string = graphite.dup
+            labels.map do |key, value|
+              rendered_string.gsub!(":#{key}", value.to_s)
+            end
+
+            rendered_string.include?(':') and warn("Graphite string #{rendered_string} not fully rendered. Expecting additional label that has not been rendered.")
+            rendered_string
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/declare_metrics/counter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Invoca
+  module Metrics
+    module Prometheus
+      module DeclareMetrics
+        class Counter < Base
+
+          def increment(value = 1, **labels)
+            if metric.present?
+              metric.increment(default_labels.merge(labels), value)
+            else
+              warn("Counter being incremented without metric being present")
+            end
+
+            if graphite
+              Invoca::Metrics::Client.metrics.count(render_graphite_string(**default_labels.merge(labels)), value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/declare_metrics/dsl.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative '../configuration'
+require_relative 'counter'
+require_relative 'gauge'
+require_relative 'histogram'
+
+module Invoca
+  module Metrics
+    module Prometheus
+      module DeclareMetrics
+        class Dsl
+
+          attr_reader :declared_metrics, :source, :prometheus_exporter_config
+
+          def initialize(source)
+            @source = source
+            @declared_metrics = []
+            if Invoca::Metrics::Prometheus.config_present?
+              @prometheus_exporter_config = Invoca::Metrics::Prometheus.config
+            end
+          end
+
+          def counter(name, labels: {}, graphite: nil, help: nil)
+            # TODO: We can drop the 'to_s' when we stop supporting ruby 2.6
+            if !name.to_s.end_with?("_total")
+              warn("#{source} counter #{name.inspect} should end in \"_total\" so that Grafana can tell that it is a counter.")
+            end
+            metric =
+              if Invoca::Metrics::Prometheus.config_present?
+                prometheus_exporter_config.register_metric(:counter, name, help, { labels: labels, graphite: graphite })
+              end
+            add_metric(Counter.new(name, :counter, source, metric, labels: labels, graphite: graphite))
+          end
+
+          def histogram(name, buckets:, labels: {}, graphite: nil, help: nil)
+            metric =
+              if Invoca::Metrics::Prometheus.config_present?
+                prometheus_exporter_config.register_metric(:histogram, name, help, { labels: labels, graphite: graphite, buckets: buckets })
+              end
+            add_metric(Histogram.new(name, :histogram, source, metric, buckets: buckets, labels: labels, graphite: graphite))
+          end
+
+          def gauge(name, labels: {}, graphite: nil, help: nil)
+            metric =
+              if Invoca::Metrics::Prometheus.config_present?
+                prometheus_exporter_config.register_metric(:gauge, name, help, { labels: labels, graphite: graphite })
+              end
+            add_metric(Gauge.new(name, :gauge, source, metric, labels: labels, graphite: graphite))
+          end
+
+          private
+
+          def add_metric(metric)
+            @declared_metrics << metric
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/declare_metrics/gauge.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'active_support/deprecation'
+
+module Invoca
+  module Metrics
+    module Prometheus
+      module DeclareMetrics
+        class Gauge < Base
+
+          def set(value = :'1', **labels)
+            if value == :'1'
+              value = 1
+              ActiveSupport::Deprecation.warn("gauge default value of 1 is deprecated; please pass an explicit value")
+            end
+
+            if metric.present?
+              metric.observe(value, default_labels.merge(labels))
+            else
+              warn("Gauge being set without metric being present")
+            end
+
+            if graphite
+              Invoca::Metrics::Client.metrics.gauge(render_graphite_string(**default_labels.merge(labels)), value)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/declare_metrics/histogram.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require 'benchmark'
+require 'active_support/deprecation'
+
+# TODO: Histogram could override Base#settings to include buckets
+module Invoca
+  module Metrics
+    module Prometheus
+      module DeclareMetrics
+        class Histogram < Base
+
+          attr_reader :buckets
+
+          def initialize(name, type, source, metric, buckets:, labels:, graphite:)
+            @buckets = buckets
+            super(name, type, source, metric, labels: labels, graphite: graphite)
+          end
+
+          def add(value = :'1', **labels)
+            if value == :'1'
+              value = 1
+              ActiveSupport::Deprecation.warn("histogram default value of 1 is deprecated; please pass an explicit value")
+            end
+
+            if metric.present?
+              metric.observe(value, default_labels.merge(labels)) if metric.present?
+            else
+              warn("Histogram being used without metric being present")
+            end
+
+            if graphite
+              Invoca::Metrics::Client.metrics.timer(render_graphite_string(**default_labels.merge(labels)), value)
+            end
+          end
+
+          def time(return_timing: false, **labels)
+            result = nil
+            elapsed_time = Benchmark.realtime { result = yield }
+            elapsed_time_ms = (elapsed_time * 1000).to_i
+            add(elapsed_time_ms, **labels)
+
+            return_timing ? [result, elapsed_time_ms] : result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus/metrics_registry.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Invoca
+  module Metrics
+    module Prometheus
+      class MetricsRegistry
+        class ContradictoryMetricRegistration < StandardError; end
+
+        attr_reader :metrics
+
+        def initialize
+          @metrics = {}
+        end
+
+        def register(metric)
+          validate_metric_name!(metric)
+          @metrics[metric.name] = metric
+        end
+
+        def method_missing(method_name, *args)
+          metrics[method_name] || super
+        end
+
+        def respond_to?(method_name, *args)
+          @metrics[method_name] || super
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          @metrics[method_name] || super
+        end
+
+        private
+
+        def validate_metric_name!(metric)
+          if (existing = @metrics[metric.name])
+            metric.settings == existing.settings or
+              raise ContradictoryMetricRegistration, "metric #{metric.name} already registered by #{existing.source} with contradictory settings #{existing.settings} vs new registered #{metric.source} with setting #{metric.settings}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/prometheus.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative './prometheus/configuration'
+require_relative './prometheus/metrics_registry'
+
+module Invoca
+  module Metrics
+    module Prometheus
+      class NotConfiguredError < StandardError; end
+
+      class << self
+        # This method is used to configure Invoca::Metrics to export metrics to be
+        # pulled into Prometheus.
+        #
+        # @yield [Invoca::Metrics::Prometheus::Configuration]
+        #
+        # @void
+        def configure
+          @config = Invoca::Metrics::Prometheus::Configuration.new.tap { |config| yield config }.freeze
+
+          # If there are any metrics at this point, it's likely that Prometheus metrics
+          # were declared before invoca-metrics was loaded; walk the set of metrics in the
+          # registry, resetting their PrometheusExporter::Metric objects
+          metrics.metrics.values.each do |metric|
+            case metric.type
+            when :counter
+              metric.reset_metric(@config.register_metric(:counter, metric.name, nil, { labels: metric.default_labels, graphite: metric.graphite }))
+            when :gauge
+              metric.reset_metric(@config.register_metric(:gauge, metric.name, nil, { labels: metric.default_labels, graphite: metric.graphite }))
+            when :histogram
+              metric.reset_metric(@config.register_metric(:histogram, metric.name, nil, { labels: metric.default_labels, graphite: metric.graphite, buckets: metric.buckets }))
+            else
+              warn("Metric type #{metric.type} not one of #{Invoca::Metrics::Prometheus::Configuration::AVAILABLE_METRIC_TYPES}.")
+            end
+          end
+
+          @config
+        end
+
+        # Accessor for the existing Prometheus configuration
+        #
+        # @raise [Invoca::Metrics::Prometheus::NotConfiguredError]
+        # @return [Invoca::Metrics::Prometheus::Configuration]
+        def config
+          @config or raise NotConfiguredError, 'Invoca::Metrics::Prometheus is trying to be used without being configured'
+        end
+
+        # Helper method for checking if configuration is present
+        #
+        # @return [Boolean]
+        def config_present?
+          @config.present?
+        end
+
+        # TODO: Rename metrics -> registry so that we don't have things like metrics.metrics
+        # (see the configure method above)
+ 
+        # This method is used to access the global list of all metrics that were registered
+        #
+        # @return [Invoca::Metrics::Prometheus::MetricsRegistry]
+        def metrics
+          @metrics ||= MetricsRegistry.new
+        end
+      end
+    end
+  end
+end

data/lib/invoca/metrics/statsd_client.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'statsd'
+
+module Invoca
+  module Metrics
+    class StatsdClient < ::Statsd
+      MILLISECONDS_IN_SECOND = 1000
+
+      @log_send_failures = true
+
+      class << self
+        attr_accessor :log_send_failures
+      end
+
+      def time(stat, sample_rate = 1)
+        start = Time.now
+        result = yield
+        length_of_time = ((Time.now - start) * MILLISECONDS_IN_SECOND).round
+        timing(stat, length_of_time, sample_rate)
+        [result, length_of_time]
+      end
+
+      def send_to_socket(message)
+        self.class.logger&.debug { "Statsd: #{message}" }
+        socket.send(message, 0)
+      rescue => ex
+        if self.class.log_send_failures
+          self.class.logger&.error { "Statsd exception sending: #{ex.class}: #{ex}" }
+        end
+
+        nil
+      end
+
+      private
+
+      # Socket connection should be Thread local and not Fiber local
+      # Can't use `Thread.current[:statsd_client] ||=` because that will be fiber-local as well.
+      def socket
+        Thread.current.thread_variable_get(:statsd_socket) || Thread.current.thread_variable_set(:statsd_socket, new_socket)
+      end
+
+      def new_socket
+        self.class.logger&.info { "Statsd client connection info -- [hostname: #{@host}, port: #{@port}]" }
+        UDPSocket.new.tap { |udp| udp.connect(@host, @port) }
+      end
+    end
+  end
+end

data/lib/invoca/metrics/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Invoca
+  module Metrics
+    VERSION = "1.14.1"
+  end
+end

data/lib/invoca/metrics.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext'
+
+require "invoca/metrics/version"
+require "invoca/metrics/statsd_client"
+require "invoca/metrics/client"
+require "invoca/metrics/direct_metric"
+require "invoca/metrics/batch"
+require "invoca/metrics/gauge_cache"
+require "invoca/metrics/prometheus"
+require "invoca/metrics/prometheus/declare_metrics/dsl"
+
+module Invoca
+  module Metrics
+    CONFIG_FIELDS = [:service_name, :server_name, :sub_server_name, :cluster_name, :statsd_host, :statsd_port].freeze
+
+    class << self
+      attr_accessor(*(CONFIG_FIELDS - [:service_name]), :default_config_key)
+      attr_writer :service_name, :graphite_usage_logging_enabled
+
+      def graphite_usage_logging_enabled
+        if @graphite_usage_logging_enabled.nil?
+          @graphite_usage_logging_enabled = true
+        end
+        @graphite_usage_logging_enabled
+      end
+
+      def service_name
+        @service_name or raise ArgumentError, "You must assign a value to Invoca::Metrics.service_name"
+      end
+
+      def initialized?
+        @service_name
+      end
+
+      def config
+        @config ||= {}
+      end
+
+      def config=(config_hash)
+        config_valid?(config_hash) or raise ArgumentError, "Invalid config #{config_hash}. Allowed fields for config key: #{CONFIG_FIELDS}."
+        @config = config_hash
+      end
+
+      def default_client_config
+        {
+          service_name:    Invoca::Metrics.service_name,
+          server_name:     Invoca::Metrics.server_name,
+          cluster_name:    Invoca::Metrics.cluster_name,
+          statsd_host:     Invoca::Metrics.statsd_host,
+          statsd_port:     Invoca::Metrics.statsd_port,
+          sub_server_name: Invoca::Metrics.sub_server_name
+        }.merge(config[default_config_key] || {})
+      end
+
+      private
+
+      def config_valid?(config_hash)
+        config_hash.nil? || config_hash.all? { |_config_key, config_key_hash| (config_key_hash.keys - CONFIG_FIELDS).empty?  }
+      end
+    end
+
+    # mix this module into your classes that need to send metrics
+    #
+    module Source
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        @metrics_namespace = nil
+
+        def metrics_namespace(namespace)
+          @metrics_namespace = namespace
+        end
+
+        def metrics
+          metrics_for(config_key: Invoca::Metrics.default_config_key)
+        end
+
+        def metrics_for(config_key:, namespace: nil)
+          config_from_key      = Invoca::Metrics.config[config_key] || {}
+          metrics_config       = if (effective_namespace = namespace || @metrics_namespace)
+                                   config_from_key.merge(namespace: effective_namespace)
+                                 else
+                                   config_from_key
+                                 end
+          Client.metrics(**Invoca::Metrics.default_client_config.merge(metrics_config))
+        end
+
+        def declare_metrics(&block)
+          dsl = Invoca::Metrics::Prometheus::DeclareMetrics::Dsl.new(self)
+          dsl.instance_eval(&block)
+          dsl.declared_metrics.each do |metric|
+            Invoca::Metrics::Prometheus.metrics.register(metric)
+          end
+        end
+
+        def prometheus_metrics
+          Invoca::Metrics::Prometheus.metrics
+        end
+      end
+
+      def prometheus_metrics
+        self.class.prometheus_metrics
+      end
+
+      def metrics
+        self.class.metrics
+      end
+
+      def metrics_for(config_key:, namespace: nil)
+        self.class.metrics_for(config_key: config_key, namespace: namespace)
+      end
+    end
+  end
+end

metadata

@@ -1,24 +1,105 @@
 --- !ruby/object:Gem::Specification
 name: invoca-metrics
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 1.14.1
 platform: ruby
 authors:
 - Invoca development
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-18 00:00:00.000000000 Z
-dependencies: []
-description:
+date: 2022-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '7'
+- !ruby/object:Gem::Dependency
+  name: statsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: prometheus_exporter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: contextual_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: Invoca metrics reporting library
 email:
 - development@invoca.com
 executables: []
 extensions: []
 extra_rdoc_files: []
-files: []
-homepage:
-licenses: []
+files:
+- LICENSE.txt
+- README.md
+- invoca-metrics.gemspec
+- lib/invoca/metrics.rb
+- lib/invoca/metrics/batch.rb
+- lib/invoca/metrics/client.rb
+- lib/invoca/metrics/direct_metric.rb
+- lib/invoca/metrics/gauge_cache.rb
+- lib/invoca/metrics/prometheus.rb
+- lib/invoca/metrics/prometheus/configuration.rb
+- lib/invoca/metrics/prometheus/declare_metrics/base.rb
+- lib/invoca/metrics/prometheus/declare_metrics/counter.rb
+- lib/invoca/metrics/prometheus/declare_metrics/dsl.rb
+- lib/invoca/metrics/prometheus/declare_metrics/gauge.rb
+- lib/invoca/metrics/prometheus/declare_metrics/histogram.rb
+- lib/invoca/metrics/prometheus/metrics_registry.rb
+- lib/invoca/metrics/statsd_client.rb
+- lib/invoca/metrics/version.rb
+homepage: https://github.com/Invoca/invoca-metrics
+licenses:
+- MIT
 metadata:
   allowed_push_host: https://rubygems.org
 post_install_message:
@@ -39,5 +120,5 @@ requirements: []
 rubygems_version: 3.1.6
 signing_key:
 specification_version: 4
-summary: Invoca metrics
+summary: Invoca metrics reporting library
 test_files: []