ddsketch 0.1.0

data/lib/ddsketch/base_sketch.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module DDSketch
+  # A quantile sketch with relative-error guarantees. This sketch computes
+  # quantile values with an approximation error that is relative to the actual
+  # quantile value. It works on both negative and non-negative input values.
+  #
+  # For instance, using DDSketch with a relative accuracy guarantee set to 1%, if
+  # the expected quantile value is 100, the computed quantile value is guaranteed to
+  # be between 99 and 101. If the expected quantile value is 1000, the computed
+  # quantile value is guaranteed to be between 990 and 1010.
+  #
+  # DDSketch works by mapping floating-point input values to bins and counting the
+  # number of values for each bin. The underlying structure that keeps track of bin
+  # counts is store.
+  #
+  # The memory size of the sketch depends on the range that is covered by the input
+  # values: the larger that range, the more bins are needed to keep track of the
+  # input values. As a rough estimate, if working on durations with a relative
+  # accuracy of 2%, about 2kB (275 bins) are needed to cover values between 1
+  # millisecond and 1 minute, and about 6kB (802 bins) to cover values between 1
+  # nanosecond and 1 day.
+  #
+  # The size of the sketch can be have a fail-safe upper-bound by using collapsing
+  # stores. As shown in
+  # <a href="http://www.vldb.org/pvldb/vol12/p2195-masson.pdf">the DDSketch paper</a>
+  # the likelihood of a store collapsing when using the default bound is vanishingly
+  # small for most data.
+  #
+  # @abstract Subclass and override to implement a custom Sketch class.
+  class BaseSketch
+    # @return [Float] the default relative accuracy for key mapping instantiation
+    DEFAULT_REL_ACC = 0.01
+
+    # @return [Integer] the default bin limit for collasping dense store instantiation
+    DEFAULT_BIN_LIMIT = 2048
+
+    # @return [Mapping::KeyMapping] Mapping between values and integer indices that imposes relative accuracy guarantees.
+    attr_reader :mapping
+
+    # @return [Store::DenseStore] store maps integers to counters
+    attr_reader :store
+
+    # @return [Store::DenseStore] store maps negative integers to counters
+    attr_reader :negative_store
+
+    # @return [Float] the count of zeros in the sketch
+    attr_reader :zero_count
+
+    # @return [Float] the maximum value in the sketch
+    attr_reader :max
+
+    # @return [Float] the minimum value in the sketch
+    attr_reader :min
+
+    # @return [Float] the sum of values in the sketch
+    attr_reader :sum
+
+    # @return [Float] the count of values in the sketch
+    attr_reader :count
+
+    # @param [Mapping::KeyMapping] mapping
+    #   mapping between values and integer indices that imposes relative accuracy guarantees.
+    # @param [Store::DenseStore] store
+    #   store maps integers to counters
+    # @param [Store::DenseStore] negative_store
+    #   store maps negative integers to counters
+    # @param [Float] zero_count
+    #   the count of zeros in the sketch
+    def initialize(mapping:, store:, negative_store:, zero_count: 0.0)
+      @mapping = mapping
+      @store = store
+      @negative_store = negative_store
+      @zero_count = zero_count
+
+      @relative_accuracy = mapping.relative_accuracy
+      @count = @negative_store.count + @zero_count + @store.count
+      @min = Float::INFINITY
+      @max = -Float::INFINITY
+      @sum = 0.0
+    end
+
+    # Average of the sketch
+    #
+    # @return [Float]
+    def avg
+      sum / count
+    end
+
+    # Add a value to the sketch.
+    #
+    # @param [Float] val The value to be added.
+    # @param [Float] weight Must be positive.
+    #
+    # @return [nil]
+    def add(val, weight = 1.0)
+      raise ArgumentError, "weight must be positive" if weight <= 0.0
+
+      if val > @mapping.min_possible
+        @store.add(@mapping.key(val), weight)
+      elsif val < -@mapping.min_possible
+        @negative_store.add(@mapping.key(-val), weight)
+      else
+        @zero_count += weight
+      end
+
+      # Keep track of summary stats
+      @count += weight
+      @sum += val * weight
+      @min = val if val < @min
+      @max = val if val > @max
+
+      nil
+    end
+
+    # Return the approximate value at the specified quantile.
+    #
+    # @param [Float] quantile Must be between 0 ~ 1
+    #
+    # @return [Float]
+    def get_quantile_value(quantile)
+      return nil if quantile < 0 || quantile > 1 || @count == 0
+
+      rank = quantile * (@count - 1)
+      if rank < @negative_store.count
+        reversed_rank = @negative_store.count - rank - 1
+        key = @negative_store.key_at_rank(reversed_rank, false)
+        quantile_value = -@mapping.value(key)
+      elsif rank < @zero_count + @negative_store.count
+        return 0
+      else
+        key = @store.key_at_rank(
+          rank - @zero_count - @negative_store.count
+        )
+        quantile_value = @mapping.value(key)
+      end
+      quantile_value
+    end
+
+    # Merge the given sketch into the current one. After this operation, this sketch
+    #   encodes the values that were added to both this and the input sketch.
+    #
+    # @param [BaseSketch] sketch The sketch to be merged.
+    #
+    # @return [nil]
+    def merge(sketch)
+      unless mergeable?(sketch)
+        raise InvalidSketchMergeError, "Cannot merge two sketches with different relative accuracy"
+      end
+
+      return if sketch.count == 0
+
+      if @count == 0
+        copy(sketch)
+        return
+      end
+
+      # Merge the stores
+      @store.merge(sketch.store)
+      @negative_store.merge(sketch.negative_store)
+      @zero_count += sketch.zero_count
+
+      # Merge summary stats
+      @count += sketch.count
+      @sum += sketch.sum
+      @min = sketch.min if sketch.min < @min
+
+      @max = sketch.max if sketch.max > @max
+
+      nil
+    end
+
+    # @return [Float] the count of values in the sketch
+    def num_values
+      @count
+    end
+
+    private
+
+    # Two sketches can be merged only if their gammas are equal.
+    def mergeable?(other)
+      @mapping.gamma == other.mapping.gamma
+    end
+
+    # Copy the input sketch into this one
+    def copy(sketch)
+      @store.copy(sketch.store)
+      @negative_store.copy(sketch.negative_store)
+      @zero_count = sketch.zero_count
+      @min = sketch.min
+      @max = sketch.max
+      @count = sketch.count
+      @sum = sketch.sum
+    end
+  end
+end

data/lib/ddsketch/errors.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module DDSketch
+  class BaseError < ::StandardError
+  end
+
+  # Error when merging two incompatible sketches
+  class InvalidSketchMergeError < BaseError
+  end
+end

data/lib/ddsketch/log_collapsing_highest_dense_sketch.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module DDSketch
+  # Implementation of BaseSketch with optimized memory usage at the cost of
+  # lower ingestion speed, using a limited number of bins. When the maximum
+  # number of bins is reached, bins with highest indices are collapsed, which
+  # causes the relative accuracy to be lost on the highest quantiles. For the
+  # default bin limit, collapsing is unlikely to occur unless the data is
+  # distributed with tails heavier than any subexponential.
+  class LogCollapsingHighestDenseSketch < BaseSketch
+    # @param relative_accuracy (see Sketch#initialize)
+    # @param [Integer] bin_limit the maximum number of bins
+    def initialize(relative_accuracy: DEFAULT_REL_ACC, bin_limit: DEFAULT_BIN_LIMIT)
+      super(
+        mapping: Mapping::LogarithmicKeyMapping.new(relative_accuracy: relative_accuracy),
+        store: Store::CollapsingHighestDenseStore.new(bin_limit: bin_limit),
+        negative_store: Store::CollapsingHighestDenseStore.new(bin_limit: bin_limit)
+      )
+    end
+  end
+end

data/lib/ddsketch/log_collapsing_lowest_dense_sketch.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module DDSketch
+  # Implementation of BaseSketch with optimized memory usage at the cost of
+  # lower ingestion speed, using a limited number of bins. When the maximum
+  # number of bins is reached, bins with lowest indices are collapsed, which
+  # causes the relative accuracy to be lost on the lowest quantiles. For the
+  # default bin limit, collapsing is unlikely to occur unless the data is
+  # distributed with tails heavier than any subexponential.
+  class LogCollapsingLowestDenseSketch < BaseSketch
+    # @param relative_accuracy (see Sketch#initialize)
+    # @param [Integer] bin_limit the maximum number of bins
+    def initialize(relative_accuracy: DEFAULT_REL_ACC, bin_limit: DEFAULT_BIN_LIMIT)
+      super(
+        mapping: Mapping::LogarithmicKeyMapping.new(relative_accuracy: relative_accuracy),
+        store: Store::CollapsingLowestDenseStore.new(bin_limit: bin_limit),
+        negative_store: Store::CollapsingLowestDenseStore.new(bin_limit: bin_limit)
+      )
+    end
+  end
+end

data/lib/ddsketch/mapping/cubically_interpolated_key_mapping.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Mapping
+    # A fast KeyMapping that approximates the memory-optimal LogarithmicMapping by
+    # extracting the floor value of the logarithm to the base 2 from the binary
+    # representations of floating-point values and cubically interpolating the
+    # logarithm in-between.
+    class CubicallyInterpolatedKeyMapping < KeyMapping
+      A = 6.0 / 35.0
+      B = -3.0 / 5.0
+      C = 10.0 / 7.0
+
+      #
+      # Indicates cubically interpolating algorithm
+      #
+      # @return [Symbol]
+      #
+      def self.interpolation
+        :cubic
+      end
+
+      # (see KeyMapping#initialize)
+      def initialize(relative_accuracy:, offset: 0.0)
+        super(relative_accuracy: relative_accuracy, offset: offset)
+
+        @multiplier /= C
+      end
+
+      protected
+
+      def log_gamma(value)
+        _cubic_log2_approx(value) * @multiplier
+      end
+
+      def pow_gamma(value)
+        _cubic_exp2_approx(value / @multiplier)
+      end
+
+      # Approximates log2 using a cubic polynomial
+      def _cubic_log2_approx(value)
+        mantissa, exponent = Math.frexp(value)
+        significand = 2 * mantissa - 1
+        (
+          (A * significand + B) * significand + C
+        ) * significand + (exponent - 1)
+      end
+
+      def _cubic_exp2_approx(value)
+        exponent = Integer(value.floor)
+        delta_0 = B * B - 3 * A * C
+
+        # Derived from Cardano's formula
+        delta_1 = (2.0 * B * B * B) - (9.0 * A * B * C) - (27.0 * A * A * (value - exponent))
+        cardano = Math.cbrt(
+          (delta_1 - ((delta_1 * delta_1 - 4 * delta_0 * delta_0 * delta_0)**0.5)) / 2.0
+        )
+
+        significand_plus_one = (
+          -(B + cardano + delta_0 / cardano) / (3.0 * A) + 1.0
+        )
+        mantissa = significand_plus_one / 2
+
+        # JRuby has inconsistent result with `Math.ldexp`
+        # https://github.com/jruby/jruby/issues/7234
+        Math.ldexp(mantissa, exponent + 1)
+      end
+    end
+  end
+end

data/lib/ddsketch/mapping/key_mapping.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Mapping
+    # A mapping between values and integer indices that imposes relative accuracy
+    # guarantees. Specifically, for any value `minIndexableValue() < value <
+    # maxIndexableValue` implementations of `KeyMapping` must be such that
+    # `value(key(v))` is close to `v` with a relative error that is less than
+    # `relative_accuracy`.
+    #
+    # In implementations of KeyMapping, there is generally a trade-off between the
+    # cost of computing the key and the number of keys that are required to cover a
+    # given range of values (memory optimality). The most memory-optimal mapping is
+    # the LogarithmicMapping, but it requires the costly evaluation of the logarithm
+    # when computing the index. Other mappings can approximate the logarithmic
+    # mapping, while being less computationally costly.
+    #
+    # @abstract Subclass and override to implement a custom KeyMapping class.
+    class KeyMapping
+      # @return [Float] the base for the exponential buckets. gamma = (1 + alpha) / (1 - alpha)
+      attr_reader :gamma
+
+      # @return [Float] the relative accuaracy guaranteed, must between 0 ~ 1
+      attr_reader :relative_accuracy
+
+      # @return [Float] the smallest value the sketch can distinguish from 0
+      attr_reader :min_possible
+
+      # @return [Float] the largest value the sketch can handle
+      attr_reader :max_possible
+
+      # @return [Float] value used to shift all bin keys
+      attr_reader :offset
+
+      #
+      # Indicates interpolating algorithm
+      #
+      # @return [Symbol, nil]
+      #
+      def self.interpolation
+        nil
+      end
+
+      # @param [Float] relative_accuracy the relative accuaracy guaranteed, must between 0 ~ 1
+      # @param [Float] offset value used to shift all bin keys
+      def initialize(relative_accuracy:, offset: 0.0)
+        if (relative_accuracy <= 0) || (relative_accuracy >= 1)
+          raise ArgumentError, "Relative accuracy must be between 0 and 1."
+        end
+
+        @relative_accuracy = relative_accuracy
+        @offset = offset
+
+        gamma_mantissa = 2 * relative_accuracy / (1 - relative_accuracy)
+
+        @gamma = 1 + gamma_mantissa
+        @multiplier = 1 / Math.log(gamma_mantissa + 1)
+        @min_possible = Float::MIN * @gamma
+        @max_possible = Float::MAX / @gamma
+      end
+
+      #
+      # Returns the key specifying the bucket for value
+      #
+      # @param [Float] value
+      #
+      # @return [Integer]
+      #
+      def key(value)
+        Integer(log_gamma(value).ceil + @offset)
+      end
+
+      #
+      # Returns the value represented by the bucket specified by the key
+      #
+      # @param [Integer] key
+      #
+      # @return [Float]
+      #
+      def value(key)
+        pow_gamma(key - @offset) * (2.0 / (1 + @gamma))
+      end
+
+      #
+      # Indicates interpolating algorithm
+      #
+      # @return [Symbol, nil]
+      #
+      def interpolation
+        self.class.interpolation
+      end
+
+      protected
+
+      def log_gamma(value)
+      end
+
+      def pow_gamma(value)
+      end
+    end
+  end
+end

data/lib/ddsketch/mapping/linear_interpolated_key_mapping.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Mapping
+    # A fast KeyMapping that approximates the memory-optimal
+    # LogarithmicMapping by extracting the floor value of the logarithm to the
+    # base 2 from the binary representations of floating-point values and
+    # linearly interpolating the logarithm in-between.
+    class LinearlyInterpolatedKeyMapping < KeyMapping
+      #
+      # Indicates linear interpolating algorithm
+      #
+      # @return [nil]
+      #
+      def self.interpolation
+        :linear
+      end
+
+      protected
+
+      def log_gamma(value)
+        _log2_approx(value) * @multiplier
+      end
+
+      def pow_gamma(value)
+        _exp2_approx(value / @multiplier)
+      end
+
+      # Approximates log2 by s + f
+      # where v = (s+1) * 2 ** f  for s in [0, 1)
+
+      # frexp(v) returns m and e s.t.
+      # v = m * 2 ** e ; (m in [0.5, 1) or 0.0)
+      # so we adjust m and e accordingly
+      def _log2_approx(value)
+        mantissa, exponent = Math.frexp(value)
+        significand = 2 * mantissa - 1
+
+        significand + (exponent - 1)
+      end
+
+      def _exp2_approx(value)
+        exponent = Integer(value.floor + 1)
+        mantissa = (value - exponent + 2) / 2.0
+
+        # JRuby has inconsistent result with `Math.ldexp`
+        # https://github.com/jruby/jruby/issues/7234
+        Math.ldexp(mantissa, exponent)
+      end
+    end
+  end
+end

data/lib/ddsketch/mapping/logarithmic_key_mapping.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Mapping
+    # A memory-optimal KeyMapping, i.e., given a targeted relative accuracy, it
+    # requires the least number of keys to cover a given range of values. This is
+    # done by logarithmically mapping floating-point values to integers.
+    class LogarithmicKeyMapping < KeyMapping
+      # (see KeyMapping#initialize)
+      def initialize(relative_accuracy:, offset: 0.0)
+        super(relative_accuracy: relative_accuracy, offset: offset)
+        @multiplier *= Math.log(2)
+      end
+
+      protected
+
+      def log_gamma(value)
+        Math.log(value, 2) * @multiplier
+      end
+
+      def pow_gamma(value)
+        2**(value / @multiplier)
+      end
+    end
+  end
+end

data/lib/ddsketch/proto/ddsketch.proto

@@ -0,0 +1,66 @@
+/* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License 2.0.
+ * This product includes software developed at Datadog (https://www.datadoghq.com/).
+ * Copyright 2020 Datadog, Inc.
+ */
+
+ syntax = "proto3";
+
+ option ruby_package = "DDSketch::Proto";
+
+ // A DDSketch is essentially a histogram that partitions the range of positive values into an infinite number of
+ // indexed bins whose size grows exponentially. It keeps track of the number of values (or possibly floating-point
+ // weights) added to each bin. Negative values are partitioned like positive values, symmetrically to zero.
+ // The value zero as well as its close neighborhood that would be mapped to extreme bin indexes is mapped to a specific
+ // counter.
+ message DDSketch {
+   // The mapping between positive values and the bin indexes they belong to.
+   IndexMapping mapping = 1;
+
+   // The store for keeping track of positive values.
+   Store positiveValues = 2;
+
+   // The store for keeping track of negative values. A negative value v is mapped using its positive opposite -v.
+   Store negativeValues = 3;
+
+   // The count for the value zero and its close neighborhood (whose width depends on the mapping).
+   double zeroCount = 4;
+ }
+
+ // How to map positive values to the bins they belong to.
+ message IndexMapping {
+   // The gamma parameter of the mapping, such that bin index that a value v belongs to is roughly equal to
+   // log(v)/log(gamma).
+   double gamma = 1;
+
+   // An offset that can be used to shift all bin indexes.
+   double indexOffset = 2;
+
+   // To speed up the computation of the index a value belongs to, the computation of the log may be approximated using
+   // the fact that the log to the base 2 of powers of 2 can be computed at a low cost from the binary representation of
+   // the input value. Other values can be approximated by interpolating between successive powers of 2 (linearly,
+   // quadratically or cubically).
+   // NONE means that the log is to be computed exactly (no interpolation).
+   Interpolation interpolation = 3;
+   enum Interpolation {
+     NONE = 0;
+     LINEAR = 1;
+     QUADRATIC = 2;
+     CUBIC = 3;
+   }
+ }
+
+ // A Store maps bin indexes to their respective counts.
+ // Counts can be encoded sparsely using binCounts, but also in a contiguous way using contiguousBinCounts and
+ // contiguousBinIndexOffset. Given that non-empty bins are in practice usually contiguous or close to one another, the
+ // latter contiguous encoding method is usually more efficient than the sparse one.
+ // Both encoding methods can be used conjointly. If a bin appears in both the sparse and the contiguous encodings, its
+ // count value is the sum of the counts in each encodings.
+ message Store {
+   // The bin counts, encoded sparsely.
+   map<sint32, double> binCounts = 1;
+
+   // The bin counts, encoded contiguously. The values of contiguousBinCounts are the counts for the bins of indexes
+   // o, o+1, o+2, etc., where o is contiguousBinIndexOffset.
+   repeated double contiguousBinCounts = 2 [packed = true];
+   sint32 contiguousBinIndexOffset = 3;
+ }

data/lib/ddsketch/proto/ddsketch_pb.rb

@@ -0,0 +1,36 @@
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: ddsketch.proto
+
+require "google/protobuf"
+
+Google::Protobuf::DescriptorPool.generated_pool.build do
+  add_message "DDSketch" do
+    optional :mapping, :message, 1, "IndexMapping"
+    optional :positiveValues, :message, 2, "Store"
+    optional :negativeValues, :message, 3, "Store"
+    optional :zeroCount, :double, 4
+  end
+  add_message "IndexMapping" do
+    optional :gamma, :double, 1
+    optional :indexOffset, :double, 2
+    optional :interpolation, :enum, 3, "IndexMapping.Interpolation"
+  end
+  add_enum "IndexMapping.Interpolation" do
+    value :NONE, 0
+    value :LINEAR, 1
+    value :QUADRATIC, 2
+    value :CUBIC, 3
+  end
+  add_message "Store" do
+    map :binCounts, :sint32, :double, 1
+    repeated :contiguousBinCounts, :double, 2
+    optional :contiguousBinIndexOffset, :sint32, 3
+  end
+end
+
+module DDSketch::Proto
+  DDSketch = Google::Protobuf::DescriptorPool.generated_pool.lookup("DDSketch").msgclass
+  IndexMapping = Google::Protobuf::DescriptorPool.generated_pool.lookup("IndexMapping").msgclass
+  IndexMapping::Interpolation = Google::Protobuf::DescriptorPool.generated_pool.lookup("IndexMapping.Interpolation").enummodule
+  Store = Google::Protobuf::DescriptorPool.generated_pool.lookup("Store").msgclass
+end

data/lib/ddsketch/proto.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "ddsketch/proto/ddsketch_pb"
+
+module DDSketch
+  # Namespace for protobuf object generated by `google-protobuf`
+  # @!visibility private
+  module Proto
+    INTERPOLATION_MAPPING = {
+      linear: IndexMapping::Interpolation::LINEAR,
+      cubic: IndexMapping::Interpolation::CUBIC
+    }.freeze
+
+    private_constant :INTERPOLATION_MAPPING
+
+    module_function
+
+    def serialize_sketch(sketch)
+      DDSketch.new(
+        mapping: serialize_key_mapping(sketch.mapping),
+        positiveValues: serialize_store(sketch.store),
+        negativeValues: serialize_store(sketch.negative_store),
+        zeroCount: sketch.zero_count
+      )
+    end
+
+    def serialize_store(store)
+      Store.new(
+        contiguousBinCounts: store.bins,
+        contiguousBinIndexOffset: store.offset
+      )
+    end
+
+    def serialize_key_mapping(mapping)
+      IndexMapping.new(
+        gamma: mapping.relative_accuracy,
+        indexOffset: mapping.offset,
+        interpolation: serialize_interpolation(mapping)
+      )
+    end
+
+    def serialize_interpolation(mapping)
+      INTERPOLATION_MAPPING.fetch(mapping.interpolation, IndexMapping::Interpolation::NONE)
+    end
+  end
+end

data/lib/ddsketch/sketch.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module DDSketch
+  # The default implementation of DDSketch, with optimized memory usage at
+  # the cost of lower ingestion speed, using an unlimited number of bins. The
+  # number of bins will not exceed a reasonable number unless the data is
+  # distributed with tails heavier than any subexponential.
+  class Sketch < BaseSketch
+    # @param [Float] relative_accuracy The guaranteed relative accuracy for sketch
+    def initialize(relative_accuracy: DEFAULT_REL_ACC)
+      super(
+        mapping: Mapping::LogarithmicKeyMapping.new(relative_accuracy: relative_accuracy),
+        store: Store::DenseStore.new,
+        negative_store: Store::DenseStore.new
+      )
+    end
+  end
+end