spectator-rb 0.1.0

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/buildViaTravis.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -e
+set -x
+
+bundle exec rake test
+bundle exec rake rubocop

data/lib/spectator.rb

@@ -0,0 +1,22 @@
+require 'spectator/version'
+require 'spectator/registry'
+require 'spectator/meter_id'
+require 'spectator/clock'
+require 'spectator/timer'
+require 'spectator/counter'
+require 'spectator/distribution_summary'
+require 'spectator/gauge'
+require 'logger'
+
+# Simple library for instrumenting code to record dimensional time series.
+module Spectator
+  class << self
+    attr_writer :logger
+
+    def logger
+      @logger ||= Logger.new($stdout).tap do |log|
+        log.progname = name
+      end
+    end
+  end
+end

data/lib/spectator/atomic_number.rb

@@ -0,0 +1,49 @@
+module Spectator
+  # Thread safe number operations
+  class AtomicNumber
+    def initialize(init)
+      @value = init
+      @lock = Mutex.new
+    end
+
+    def set(value)
+      @lock.synchronize { @value = value }
+    end
+
+    def get
+      @lock.synchronize { @value }
+    end
+
+    def get_and_set(value)
+      @lock.synchronize do
+        tmp = @value
+        @value = value
+        tmp
+      end
+    end
+
+    def get_and_add(amount)
+      @lock.synchronize do
+        tmp = @value
+        @value += amount
+        tmp
+      end
+    end
+
+    def add_and_get(amount)
+      @lock.synchronize { @value += amount }
+    end
+
+    def max(value)
+      v = value.to_f
+      @lock.synchronize do
+        @value = v if v > @value || @value.nan?
+      end
+      @value
+    end
+
+    def to_s
+      "AtomicNumber{#{@value}}"
+    end
+  end
+end

data/lib/spectator/clock.rb

@@ -0,0 +1,57 @@
+module Spectator
+  # A timing source that can be used to access the current time as an object,
+  # and a high resolution monotonic time
+  class SystemClock
+    # A monotonically increasing number of nanoseconds. This is useful for
+    # recording times, or benchmarking.
+    # Note that this is not guaranteed to be steady.
+    # In other words each tick of the underlying clock may not
+    # be the same length (e.g. some seconds might be longer than others)
+    # @return A monotonic number of nanoseconds
+    def monotonic_time
+      MonotonicTime.time_in_nanoseconds
+    end
+
+    # @return a time object for the current time
+    def wall_time
+      Time.now
+    end
+  end
+
+  # A timing source useful in unit tests that can be used to mock the methods in
+  # SystemClock
+  class ManualClock
+    attr_accessor :wall_time
+    attr_accessor :monotonic_time
+
+    # Get a new object using 2000-1-1 0:0:0 UTC as the default time,
+    # and 0 nanoseconds as the number of nanos reported by monotonic_time
+    def initialize(wall_init: Time.utc(2000, 'jan', 1, 0, 0, 0), mono_time: 0)
+      @wall_time = wall_init
+      @monotonic_time = mono_time
+    end
+  end
+
+  # Gather a monotonically increasing number of nanoseconds.
+  # If Process::CLOCK_MONOTONIC is available  we use that, otherwise we attempt
+  # to use java.lang.System.nanoTime if running in jruby, and fallback
+  # to the Time.now implementation
+  module MonotonicTime
+    module_function
+
+    if defined? Process::CLOCK_MONOTONIC
+      def time_in_nanoseconds
+        Process.clock_gettime(Process::CLOCK_MONOTONIC, :nanosecond)
+      end
+    elsif (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') == 'jruby'
+      def time_in_nanoseconds
+        java.lang.System.nanoTime
+      end
+    else
+      def time_in_nanoseconds
+        t = Time.now
+        t.to_i * 10**9 + t.nsec
+      end
+    end
+  end
+end

data/lib/spectator/counter.rb

@@ -0,0 +1,34 @@
+require 'spectator/atomic_number'
+require 'spectator/measure'
+
+module Spectator
+  # A counter is used to measure the rate at which an event is occurring
+  class Counter
+    # Initialize a new instance setting its id, and starting the
+    # count at 0
+    def initialize(id)
+      @id = id
+      @count = AtomicNumber.new(0)
+    end
+
+    # Increment the counter by delta
+    def increment(delta = 1)
+      @count.add_and_get(delta)
+    end
+
+    # Get the current count as a list of Measure and reset the count to 0
+    def measure
+      [Measure.new(@id.with_stat('count'), @count.get_and_set(0))]
+    end
+
+    # Read the current count. Calls to measure will reset it
+    def count
+      @count.get
+    end
+
+    # Get a string representation for debugging purposes
+    def to_s
+      "Counter{id=#{@id}, count=#{@count.get}}"
+    end
+  end
+end

data/lib/spectator/distribution_summary.rb

@@ -0,0 +1,53 @@
+require 'spectator/atomic_number'
+
+module Spectator
+  # Track the sample distribution of events. An example would be the response
+  # sizes for requests hitting an http server.
+  #
+  # The class will report measurements for the total amount, the count, max,
+  # and the  total of the square of the amounts recorded
+  # (useful for computing a standard deviation)
+  class DistributionSummary
+    # Initialize a new DistributionSummary instance with a given id
+    def initialize(id)
+      @id = id
+      @count = AtomicNumber.new(0)
+      @total_amount = AtomicNumber.new(0)
+      @total_sq = AtomicNumber.new(0)
+      @max = AtomicNumber.new(Float::NAN)
+    end
+
+    # Update the statistics kept by the summary with the specified amount.
+    def record(amount)
+      return if amount < 0
+      @count.add_and_get(1)
+      @total_amount.add_and_get(amount)
+      @total_sq.add_and_get(amount * amount)
+      @max.max(amount)
+    end
+
+    # Get the current amount
+    def count
+      @count.get
+    end
+
+    # Return the total amount
+    def total_amount
+      @total_amount.get
+    end
+
+    # Get a list of measurements, and reset the stats
+    # The stats returned are the current count, the total amount,
+    # the sum of the square of the amounts recorded, and the max value
+    def measure
+      cnt = Measure.new(@id.with_stat('count'), @count.get_and_set(0))
+      tot = Measure.new(@id.with_stat('totalAmount'),
+                        @total_amount.get_and_set(0))
+      tot_sq = Measure.new(@id.with_stat('totalOfSquares'),
+                           @total_sq.get_and_set(0))
+      mx = Measure.new(@id.with_stat('max'), @max.get_and_set(Float::NAN))
+
+      [cnt, tot, tot_sq, mx]
+    end
+  end
+end

data/lib/spectator/gauge.rb

@@ -0,0 +1,34 @@
+require 'spectator/atomic_number'
+require 'spectator/measure'
+
+module Spectator
+  # A meter with a single value that can only be sampled at a point in time.
+  # A typical example is a queue size.
+  class Gauge
+    # Initialize a new instance of a Gauge with the given id
+    def initialize(id)
+      @id = id
+      @value = AtomicNumber.new(Float::NAN)
+    end
+
+    # Get the current value
+    def get
+      @value.get
+    end
+
+    # Set the current value to the number specified
+    def set(value)
+      @value.set(value)
+    end
+
+    # Get the current value, and reset it
+    def measure
+      [Measure.new(@id.with_stat('gauge'), @value.get_and_set(Float::NAN))]
+    end
+
+    # A string representation of this gauge, useful for debugging purposes
+    def to_s
+      "Gauge{id=#{@id}, value=#{@value.get}}"
+    end
+  end
+end

data/lib/spectator/http.rb

@@ -0,0 +1,30 @@
+require 'json'
+require 'net/http'
+
+module Spectator
+  # Helper for HTTP requests
+  class Http
+    # Create a new instance using the given registry
+    # to record stats for the requests performed
+    def initialize(registry)
+      @registry = registry
+    end
+
+    # Send a JSON payload to a given endpoing
+    def post_json(endpoint, payload)
+      s = payload.to_json
+      uri = URI(endpoint)
+      http = Net::HTTP.new(uri.host, uri.port)
+      req = Net::HTTP::Post.new(uri.path, 'Content-Type' => 'application/json')
+      req.body = s
+      begin
+        res = http.request(req)
+      rescue StandardError => e
+        Spectator.logger.info("Cause #{e.cause} - msg=#{e.message}")
+        return 400
+      end
+
+      res.value
+    end
+  end
+end

data/lib/spectator/measure.rb

@@ -0,0 +1,24 @@
+module Spectator
+  # This immutable class represents a measurement sampled from a meter
+  class Measure
+    attr_reader :id, :value
+
+    # A meter id and a value
+    def initialize(id, value)
+      @id = id
+      @value = value.to_f
+    end
+
+    # A string representation of this measurement, for debugging purposes
+    def to_s
+      "Measure{id=#{@id}, value=#{@value}}"
+    end
+
+    # Compare this measurement against another one,
+    # taking into account nan values
+    def ==(other)
+      @id == other.id && (@value == other.value ||
+          @value.nan? && other.value.nan?)
+    end
+  end
+end

data/lib/spectator/meter_id.rb

@@ -0,0 +1,52 @@
+module Spectator
+  # Identifier for a meter or Measure
+  class MeterId
+    attr_reader :name, :tags
+    def initialize(name, maybe_tags = nil)
+      tags = maybe_tags.nil? ? {} : maybe_tags
+      @name = name.to_sym
+      @tags = {}
+      tags.each { |k, v| @tags[k.to_sym] = v.to_sym }
+      @tags.freeze
+      @key = nil
+    end
+
+    # Create a new MeterId with a given key and value
+    def with_tag(key, value)
+      new_tags = @tags.dup
+      new_tags[key] = value
+      MeterId.new(@name, new_tags)
+    end
+
+    # Create a new MeterId with key=statistic and the given value
+    def with_stat(stat_value)
+      with_tag(:statistic, stat_value)
+    end
+
+    # lazyily compute a key to be used in hashes for efficiency
+    def key
+      if @key.nil?
+        hash_key = @name.to_s
+        @key = hash_key
+        keys = @tags.keys
+        keys.sort
+        keys.each do |k|
+          v = tags[k]
+          hash_key += "|#{k}|#{v}"
+        end
+        @key = hash_key
+      end
+      @key
+    end
+
+    # A string representation for debugging purposes
+    def to_s
+      "MeterId{name=#{@name}, tags=#{@tags}}"
+    end
+
+    # Compare our id and tags against another MeterId
+    def ==(other)
+      other.name == @name && other.tags == @tags
+    end
+  end
+end

data/lib/spectator/registry.rb

@@ -0,0 +1,298 @@
+require 'spectator/clock'
+require 'spectator/counter'
+require 'spectator/distribution_summary'
+require 'spectator/gauge'
+require 'spectator/http'
+require 'spectator/meter_id'
+require 'spectator/timer'
+
+module Spectator
+  # Registry to manage a set of meters
+  class Registry
+    attr_reader :config, :clock, :publisher, :common_tags
+
+    # Initialize the registry using the given config, and clock
+    # The default clock is the SystemClock
+    # The config is a Hash which should include:
+    #  :common_tags as a hash with tags that will be added to all metrics
+    #  :frequency the interval at which metrics will be sent to an
+    #  aggregator service, expressed in seconds
+    #  :uri the endpoint for the aggregator service
+    def initialize(config, clock = SystemClock.new)
+      @config = config
+      @clock = clock
+      @meters = {}
+      @common_tags = to_symbols(config[:common_tags]) || {}
+      @lock = Mutex.new
+      @publisher = Publisher.new(self)
+    end
+
+    # Create a new MeterId with the given name, and optional tags
+    def new_id(name, tags = nil)
+      MeterId.new(name, tags)
+    end
+
+    # Create or get a Counter with the given id
+    def counter_with_id(id)
+      new_meter(id) { |meter_id| Counter.new(meter_id) }
+    end
+
+    # Create or get a Gauge with the given id
+    def gauge_with_id(id)
+      new_meter(id) { |meter_id| Gauge.new(meter_id) }
+    end
+
+    # Create or get a DistributionSummary with the given id
+    def distribution_summary_with_id(id)
+      new_meter(id) { |meter_id| DistributionSummary.new(meter_id) }
+    end
+
+    # Create or get a Timer with the given id
+    def timer_with_id(id)
+      new_meter(id) { |meter_id| Timer.new(meter_id) }
+    end
+
+    # Create or get a Counter with the given name, and optional tags
+    def counter(name, tags = nil)
+      counter_with_id(MeterId.new(name, tags))
+    end
+
+    # Create or get a Gauge with the given name, and optional tags
+    def gauge(name, tags = nil)
+      gauge_with_id(MeterId.new(name, tags))
+    end
+
+    # Create or get a DistributionSummary with the given name, and optional tags
+    def distribution_summary(name, tags = nil)
+      distribution_summary_with_id(MeterId.new(name, tags))
+    end
+
+    # Create or get a Timer with the given name, and optional tags
+    def timer(name, tags = nil)
+      timer_with_id(MeterId.new(name, tags))
+    end
+
+    # Get the list of measurements from all registered meters
+    def measurements
+      @lock.synchronize do
+        @meters.values.flat_map(&:measure)
+      end
+    end
+
+    # Start publishing measurements to the aggregator service
+    def start
+      @publisher.start
+    end
+
+    # Stop publishing measurements
+    def stop
+      @publisher.stop
+    end
+
+    private
+
+    def to_symbols(tags)
+      return nil if tags.nil?
+
+      symbolic_tags = {}
+      tags.each { |k, v| symbolic_tags[k.to_sym] = v.to_sym }
+      symbolic_tags
+    end
+
+    def new_meter(meter_id)
+      @lock.synchronize do
+        meter = @meters[meter_id.key]
+        if meter.nil?
+          meter = yield(meter_id)
+          @meters[meter_id.key] = meter
+        end
+        meter
+      end
+    end
+  end
+
+  # Internal class used to publish measurements to an aggregator service
+  class Publisher
+    def initialize(registry)
+      @registry = registry
+      @started = false
+      @should_stop = false
+      @frequency = registry.config[:frequency] || 5
+      @http = Http.new(registry)
+    end
+
+    def should_start?
+      if @started
+        Spectator.logger.info('Ignoring start request. ' \
+          'Spectator registry already started')
+        return false
+      end
+
+      @started = true
+      uri = @registry.config[:uri]
+      if uri.nil? || uri.empty?
+        Spectator.logger.info('Ignoring start request since Spectator ' \
+                                  'registry has no valid uri')
+        return false
+      end
+
+      true
+    end
+
+    # Start publishing if the config is acceptable:
+    #  uri is non-nil or empty
+    def start
+      return unless should_start?
+
+      Spectator.logger.info 'Starting Spectator registry'
+
+      @should_stop = false
+      @publish_thread = Thread.new do
+        publish
+      end
+    end
+
+    # Stop publishing measurements
+    def stop
+      unless @started
+        Spectator.logger.info('Attemping to stop Spectator ' \
+          'without a previous call to start')
+        return
+      end
+
+      @should_stop = true
+      Spectator.logger.info('Stopping spectator')
+      @publish_thread.kill if @publish_thread
+
+      @started = false
+      Spectator.logger.info('Sending last batch of metrics before exiting')
+      send_metrics_now
+    end
+
+    ADD_OP = 0
+    MAX_OP = 10
+    UNKNOWN_OP = -1
+    OPS = { count: ADD_OP,
+            totalAmount: ADD_OP,
+            totalTime: ADD_OP,
+            totalOfSquares: ADD_OP,
+            percentile: ADD_OP,
+            max: MAX_OP,
+            gauge: MAX_OP,
+            activeTasks: MAX_OP,
+            duration: MAX_OP }.freeze
+    # Get the operation to be used for the given Measure
+    # Gauges are aggregated using MAX_OP, counters with ADD_OP
+    def op_for_measurement(measure)
+      stat = measure.id.tags.fetch(:statistic, :unknown)
+      OPS.fetch(stat, UNKNOWN_OP)
+    end
+
+    # Gauges are sent if they have a value
+    # Counters if they have a number of increments greater than 0
+    def should_send(measure)
+      op = op_for_measurement(measure)
+      return measure.value > 0 if op == ADD_OP
+      return !measure.value.nan? if op == MAX_OP
+
+      false
+    end
+
+    # Build a string table from the list of measurements
+    # Unique words are identified, and assigned a number starting from 0 based
+    # on their lexicographical order
+    def build_string_table(measurements)
+      common_tags = @registry.common_tags
+      table = {}
+      common_tags.each do |k, v|
+        table[k] = 0
+        table[v] = 0
+      end
+      table[:name] = 0
+      measurements.each do |m|
+        table[m.id.name] = 0
+        m.id.tags.each do |k, v|
+          table[k] = 0
+          table[v] = 0
+        end
+      end
+      keys = table.keys.sort
+      keys.each_with_index do |str, index|
+        table[str] = index
+      end
+      table
+    end
+
+    # Add a measurement to our payload table.
+    # The serialization for a measurement is:
+    #  - length of tags
+    #  - indexes for the tags based on the string table
+    #  - operation (add (0), max (10))
+    #  - floating point value
+    def append_measurement(payload, table, measure)
+      op = op_for_measurement(measure)
+      common_tags = @registry.common_tags
+      tags = measure.id.tags
+      len = tags.length + 1 + common_tags.length
+      payload.push(len)
+      common_tags.each do |k, v|
+        payload.push(table[k])
+        payload.push(table[v])
+      end
+      tags.each do |k, v|
+        payload.push(table[k])
+        payload.push(table[v])
+      end
+      payload.push(table[:name])
+      payload.push(table[measure.id.name])
+      payload.push(op)
+      payload.push(measure.value)
+    end
+
+    # Generate a payload from the list of measurements
+    # The payload is an array, with the number of elements in the string table
+    # The string table, and measurements
+    def payload_for_measurements(measurements)
+      table = build_string_table(measurements)
+      payload = []
+      payload.push(table.length)
+      strings = table.keys.sort
+      payload.concat(strings)
+      measurements.each { |m| append_measurement(payload, table, m) }
+      payload
+    end
+
+    # Get a list of measurements that should be sent
+    def registry_measurements
+      @registry.measurements.select { |m| should_send(m) }
+    end
+
+    # Send the current measurements to our aggregator service
+    def send_metrics_now
+      ms = registry_measurements
+      if ms.empty?
+        Spectator.logger.debug 'No measurements to send'
+      else
+        payload = payload_for_measurements(ms)
+        uri = @registry.config[:uri]
+        Spectator.logger.info "Sending #{ms.length} measurements to #{uri}"
+        @http.post_json(uri, payload)
+      end
+    end
+
+    # Publish loop:
+    #   send measurements to the aggregator endpoint ':uri',
+    #   every ':frequency' seconds
+    def publish
+      clock = @registry.clock
+      until @should_stop
+        start = clock.wall_time
+        Spectator.logger.info 'Publishing'
+        send_metrics_now
+        elapsed = clock.wall_time - start
+        sleep @frequency - elapsed if elapsed < @frequency
+      end
+      Spectator.logger.info 'Stopping publishing thread'
+    end
+  end
+end