ddsketch 0.1.0

data/lib/ddsketch/store/collapsing_highest_dense_store.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Store
+    # A dense store that keeps all the bins between the bin for the min_key and the
+    # bin for the max_key, but collapsing the right-most bins if the number of bins
+    # exceeds the bin_limit
+    class CollapsingHighestDenseStore < DenseStore
+      # @return [Integer] the maximum number of bins
+      attr_reader :bin_limit
+
+      # @return [Boolean] whether the store has been collapsed
+      attr_reader :is_collapsed
+
+      # @param [Integer] bin_limit the maximum number of bins
+      # @param [Integer] chunk_size the number of bins to grow by
+      def initialize(bin_limit:, chunk_size: CHUNK_SIZE)
+        super(chunk_size: chunk_size)
+
+        @bin_limit = bin_limit
+        @is_collapsed = false
+      end
+
+      # Copies the input store into the current store
+      #
+      # @param [Store::CollapsingHighestDenseStore] store the store to be copied
+      #
+      # @return [nil]
+      def copy(store)
+        super(store)
+
+        self.bin_limit = store.bin_limit
+        self.is_collapsed = store.is_collapsed
+
+        nil
+      end
+
+      # Merge another store into the current store.
+      #   collapsing the right-most bins if the number of bins
+      #   exceeds the bin_limit
+      #
+      # @param [Store::CollapsingHighestDenseStore] store
+      #   the store to be merged
+      #
+      # @return [nil]
+      def merge(store)
+        return if store.count == 0
+
+        if count == 0
+          copy(store)
+          return
+        end
+
+        extend_range(store.min_key, store.max_key) if (store.min_key < min_key) || (store.max_key > max_key)
+
+        collapse_end_idx = store.max_key - store.offset + 1
+        collapse_start_idx = [max_key + 1, store.min_key].max - store.offset
+        if collapse_end_idx > collapse_start_idx
+          collapse_count = store.bins[collapse_start_idx...collapse_end_idx].inject(:+) || 0
+          bins[-1] += collapse_count
+        else
+          collapse_start_idx = collapse_end_idx
+        end
+
+        (store.min_key).upto(collapse_start_idx + store.offset - 1).each do |key|
+          bins[key - offset] += store.bins[key - store.offset]
+        end
+
+        self.count += store.count
+
+        nil
+      end
+
+      protected
+
+      attr_writer :bin_limit, :is_collapsed
+
+      private
+
+      def get_new_length(new_min_key, new_max_key)
+        desired_length = new_max_key - new_min_key + 1
+        # For some reason mypy can't infer that min(int, int) is an int, so cast it.
+        [
+          chunk_size * (desired_length.to_f / chunk_size).ceil,
+          bin_limit
+        ].min
+      end
+
+      # Calculate the bin index for the key, extending the range if necessary
+      def get_index(key)
+        if key > max_key
+          return length - 1 if is_collapsed
+
+          extend_range(key)
+          return length - 1 if is_collapsed
+        elsif key < min_key
+          extend_range(key)
+        end
+        key - offset
+      end
+
+      # Override. Adjust the bins, the offset, the min_key, and max_key, without
+      # resizing the bins, in order to try making it fit the specified
+      # range. Collapse to the left if necessary.
+      def adjust(new_min_key, new_max_key)
+        if new_max_key - new_min_key + 1 > length
+          # The range of keys is too wide, the lowest bins need to be collapsed.
+          new_max_key = new_min_key + length - 1
+
+          if new_max_key <= min_key
+            # put everything in the last bin
+            self.offset = new_min_key
+            self.max_key = new_max_key
+            self.bins = [0.0] * length
+            bins[-1] = count
+          else
+            shift = offset - new_min_key
+
+            if shift > 0
+              collapse_start_index = new_max_key - offset + 1
+              collapse_end_index = max_key - offset + 1
+              collapsed_count = bins[collapse_start_index...collapse_end_index].inject(:+) || 0
+
+              bins[collapse_start_index...collapse_end_index] = [0.0] * (max_key - new_max_key)
+              bins[collapse_start_index - 1] += collapsed_count
+            end
+
+            self.max_key = new_max_key
+            # shift the buckets to make room for new_max_key
+            shift_bins(shift)
+          end
+
+          self.min_key = new_min_key
+          self.is_collapsed = true
+        else
+          center_bins(new_min_key, new_max_key)
+          self.min_key = new_min_key
+          self.max_key = new_max_key
+        end
+      end
+    end
+  end
+end

data/lib/ddsketch/store/collapsing_lowest_dense_store.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Store
+    # A dense store that keeps all the bins between the bin for the min_key and the
+    # bin for the max_key, but collapsing the left-most bins if the number of bins
+    # exceeds the bin_limit
+    class CollapsingLowestDenseStore < DenseStore
+      # @return [Integer] the maximum number of bins
+      attr_reader :bin_limit
+
+      # @return [Boolean] whether the store has been collapsed
+      attr_reader :is_collapsed
+
+      # @param [Integer] bin_limit the maximum number of bins
+      # @param [Integer] chunk_size the number of bins to grow by
+      def initialize(bin_limit:, chunk_size: CHUNK_SIZE)
+        super(chunk_size: chunk_size)
+
+        @bin_limit = bin_limit
+        @is_collapsed = false
+      end
+
+      # Copies the input store into the current store
+      #
+      # @param [Store::CollapsingLowestDenseStore] store the store to be copied
+      #
+      # @return [nil]
+      def copy(store)
+        super(store)
+
+        self.bin_limit = store.bin_limit
+        self.is_collapsed = store.is_collapsed
+
+        nil
+      end
+
+      # Merge another store into the current store.
+      #   collapsing the left-most bins if the number of bins
+      #   exceeds the bin_limit
+      #
+      # @param [Store::CollapsingLowestDenseStore] store
+      #   the store to be merged
+      #
+      # @return [nil]
+      def merge(store)
+        return if store.count == 0
+
+        if count == 0
+          copy(store)
+          return
+        end
+
+        extend_range(store.min_key, store.max_key) if store.min_key < min_key || store.max_key > max_key
+
+        collapse_start_idx = store.min_key - store.offset
+        collapse_end_idx = [min_key, store.max_key + 1].min - store.offset
+
+        if collapse_end_idx > collapse_start_idx
+          collapse_count = store.bins[collapse_start_idx...collapse_end_idx].inject(:+) || 0
+          bins[0] += collapse_count
+        else
+          collapse_end_idx = collapse_start_idx
+        end
+
+        (collapse_end_idx + store.offset).upto(store.max_key).each do |key|
+          bins[key - offset] += store.bins[key - store.offset]
+        end
+
+        self.count += store.count
+
+        nil
+      end
+
+      protected
+
+      attr_writer :bin_limit, :is_collapsed
+
+      private
+
+      def get_new_length(new_min_key, new_max_key)
+        desired_length = new_max_key - new_min_key + 1
+
+        [
+          chunk_size * (desired_length.to_f / chunk_size).ceil,
+          bin_limit
+        ].min
+      end
+
+      # Calculate the bin index for the key, extending the range if necessary.
+      def get_index(key)
+        if key < min_key
+          return 0 if is_collapsed
+
+          extend_range(key)
+          return 0 if is_collapsed
+        elsif key > max_key
+          extend_range(key)
+        end
+
+        key - offset
+      end
+
+      # Override. Adjust the bins, the offset, the min_key, and max_key, without
+      # resizing the bins, in order to try making it fit the specified
+      # range. Collapse to the left if necessary.
+      def adjust(new_min_key, new_max_key)
+        if new_max_key - new_min_key + 1 > length
+          # The range of keys is too wide, the lowest bins need to be collapsed.
+          new_min_key = new_max_key - length + 1
+
+          if new_min_key >= max_key
+            # put everything in the first bin
+            self.offset = new_min_key
+            self.min_key = new_min_key
+            self.bins = [0.0] * length
+            bins[0] = count
+          else
+            shift = offset - new_min_key
+
+            if shift < 0
+              collapse_start_index = min_key - offset
+              collapse_end_index = new_min_key - offset
+              collapsed_count = bins[collapse_start_index...collapse_end_index].inject(:+) || 0
+
+              bins[collapse_start_index...collapse_end_index] = [0.0] * (new_min_key - min_key)
+              bins[collapse_end_index] += collapsed_count
+            end
+
+            self.min_key = new_min_key
+            # shift the buckets to make room for new_min_key
+            shift_bins(shift)
+          end
+
+          self.max_key = new_max_key
+          self.is_collapsed = true
+        else
+          center_bins(new_min_key, new_max_key)
+          self.min_key = new_min_key
+          self.max_key = new_max_key
+        end
+      end
+    end
+  end
+end

data/lib/ddsketch/store/dense_store.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Store
+    #
+    # Stores map integers to counters. They can be seen as a collection of bins.
+    # We start with 128 bins and grow the store in chunks of 128 unless specified
+    # otherwise.
+    #
+    # A dense store that keeps all the bins between the bin for the min_key and the
+    # bin for the max_key.
+    #
+    class DenseStore
+      CHUNK_SIZE = 128
+
+      # @return [Integer] the sum of the counts for the bins
+      attr_reader :count
+
+      # @return [Integer] the minimum key bin
+      attr_reader :min_key
+
+      # @return [Integer] the maximum key bin
+      attr_reader :max_key
+
+      # @return [Integer] the number of bins to grow by
+      attr_reader :chunk_size
+
+      # @return [Integer] the difference btw the keys and the index in which they are stored
+      attr_reader :offset
+
+      # @return [Array<Float>] the bins
+      attr_reader :bins
+
+      # @param [Integer] chunk_size the number of bins to grow by
+      def initialize(chunk_size: CHUNK_SIZE)
+        @count = 0
+        @min_key = Float::INFINITY
+        @max_key = -Float::INFINITY
+
+        @chunk_size = chunk_size
+        @offset = 0
+        @bins = []
+      end
+
+      #
+      # Copies the input store into the current store
+      #
+      # @param [Store::DenseStore] store the store to be copied
+      #
+      # @return [nil]
+      #
+      def copy(store)
+        self.bins = store.bins.dup
+        self.count = store.count
+        self.min_key = store.min_key
+        self.max_key = store.max_key
+        self.offset = store.offset
+
+        nil
+      end
+
+      #
+      # Merge another store into the current store. This should be equivalent as running the
+      #   add operations that have been run on the other store on this one.
+      #
+      # @param [Store::DenseStore] store
+      #   the store to be merged
+      #
+      # @return [nil]
+      #
+      def merge(store)
+        return if store.count == 0
+
+        if count == 0
+          copy(store)
+          return
+        end
+
+        extend_range(store.min_key, store.max_key) if store.min_key < min_key || store.max_key > max_key
+
+        store.min_key.upto(store.max_key).each do |key|
+          bins[key - offset] += store.bins[key - store.offset]
+        end
+
+        self.count += store.count
+
+        nil
+      end
+
+      #
+      # Return the number of bins
+      #
+      # @return [Integer] the number of bins
+      #
+      def length
+        bins.length
+      end
+
+      #
+      # Updates the counter at the specified index key, growing the number of bins if necessary.
+      #
+      # @param [Float] key
+      # @param [Float] weight
+      #
+      # @return [nil]
+      #
+      def add(key, weight = 1.0)
+        idx = get_index(key)
+        bins[idx] += weight
+        self.count += weight
+
+        nil
+      end
+
+      #
+      # Return the key for the value at a given rank
+      #
+      # @param [Float] rank
+      # @param [Boolean] lower
+      #
+      # @return [Integer]
+      #
+      def key_at_rank(rank, lower = true)
+        running_ct = 0.0
+
+        bins.each_with_index do |bin_ct, i|
+          running_ct += bin_ct
+
+          ## ??
+          return i + offset if (lower && running_ct > rank) || (!lower && running_ct >= rank + 1)
+        end
+
+        # Never here....??
+        max_key
+      end
+
+      protected
+
+      attr_writer :count, :min_key, :max_key, :chunk_size, :offset, :bins
+
+      private
+
+      # Calculate the bin index for the key, extending the range if necessary.
+      def get_index(key)
+        extend_range(key) if key < min_key || key > max_key
+
+        key - offset
+      end
+
+      def get_new_length(new_min_key, new_max_key)
+        desired_length = new_max_key - new_min_key + 1
+
+        chunk_size * (desired_length.to_f / chunk_size).ceil
+      end
+
+      # Grow the bins as necessary and call adjust
+      def extend_range(key, _second_key = nil) # rubocop:todo Lint/UnderscorePrefixedVariableName
+        second_key = _second_key || key
+        new_min_key = [key, second_key, min_key].min
+        new_max_key = [key, second_key, max_key].max
+
+        if length == 0
+          # initialize bins
+          self.bins = [0.0] * get_new_length(new_min_key, new_max_key)
+          self.offset = new_min_key
+          adjust(new_min_key, new_max_key)
+
+        elsif new_min_key >= min_key && (new_max_key < (offset + length))
+          # no need to change the range; just update min/max keys
+          self.min_key = new_min_key
+          self.max_key = new_max_key
+
+        else
+          # grow the bins
+          new_length = get_new_length(new_min_key, new_max_key)
+          bins.push(*([0.0] * (new_length - length))) if new_length > length
+          adjust(new_min_key, new_max_key)
+        end
+      end
+
+      # Adjust the bins, the offset, the min_key, and max_key, without resizing the
+      # bins, in order to try making it fit the specified range.
+      def adjust(new_min_key, new_max_key)
+        center_bins(new_min_key, new_max_key)
+
+        self.min_key = new_min_key
+        self.max_key = new_max_key
+      end
+
+      # Shift the bins; this changes the offset.
+      def shift_bins(shift)
+        if shift > 0
+          self.bins = bins[0...-shift]
+          bins.unshift(*([0.0] * shift))
+        else
+          self.bins = bins[(shift.abs)..-1]
+          bins.push(*([0.0] * shift.abs))
+        end
+        self.offset -= shift
+      end
+
+      # Center the bins; this changes the offset.
+      def center_bins(new_min_key, new_max_key)
+        middle_key = new_min_key + (new_max_key - new_min_key + 1).div(2)
+
+        shift_bins(offset + length.div(2) - middle_key)
+      end
+    end
+  end
+end

data/lib/ddsketch/version.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module DDSketch
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 0
+    PRE = nil
+
+    def self.to_s
+      [MAJOR, MINOR, PATCH, PRE].compact.join(".")
+    end
+  end
+end

data/lib/ddsketch.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "ddsketch/version"
+require "ddsketch/errors"
+
+# sketchs
+require "ddsketch/base_sketch"
+require "ddsketch/sketch"
+require "ddsketch/log_collapsing_lowest_dense_sketch"
+require "ddsketch/log_collapsing_highest_dense_sketch"
+
+# key mappings
+require "ddsketch/mapping/key_mapping"
+require "ddsketch/mapping/logarithmic_key_mapping"
+require "ddsketch/mapping/linear_interpolated_key_mapping"
+require "ddsketch/mapping/cubically_interpolated_key_mapping"
+
+# dense stores
+require "ddsketch/store/dense_store"
+require "ddsketch/store/collapsing_lowest_dense_store"
+require "ddsketch/store/collapsing_highest_dense_store"
+
+# Namespace for DDSketch library
+module DDSketch
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: ddsketch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Datadog, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2022-07-07 00:00:00.000000000 Z
+dependencies: []
+description: DDSketch is a fast-to-insert, fully mergeable, space-efficient quantile
+  sketch with relative error guarantees.
+email:
+- dev@datadoghq.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- ".simplecov"
+- ".yardopts"
+- CHANGELOG.md
+- CONTRIBUTING.md
+- Gemfile
+- LICENSE
+- LICENSE-3rdparty.csv
+- NOTICE
+- README.md
+- Rakefile
+- ddsketch-ruby.gemspec
+- lib/ddsketch.rb
+- lib/ddsketch/base_sketch.rb
+- lib/ddsketch/errors.rb
+- lib/ddsketch/log_collapsing_highest_dense_sketch.rb
+- lib/ddsketch/log_collapsing_lowest_dense_sketch.rb
+- lib/ddsketch/mapping/cubically_interpolated_key_mapping.rb
+- lib/ddsketch/mapping/key_mapping.rb
+- lib/ddsketch/mapping/linear_interpolated_key_mapping.rb
+- lib/ddsketch/mapping/logarithmic_key_mapping.rb
+- lib/ddsketch/proto.rb
+- lib/ddsketch/proto/ddsketch.proto
+- lib/ddsketch/proto/ddsketch_pb.rb
+- lib/ddsketch/sketch.rb
+- lib/ddsketch/store/collapsing_highest_dense_store.rb
+- lib/ddsketch/store/collapsing_lowest_dense_store.rb
+- lib/ddsketch/store/dense_store.rb
+- lib/ddsketch/version.rb
+homepage: https://github.com/DataDog/ddsketch-ruby
+licenses:
+- Apache-2.0
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/DataDog/ddsketch-ruby
+  source_code_uri: https://github.com/DataDog/ddsketch-ruby
+  changelog_uri: https://github.com/DataDog/ddsketch-ruby/blob/main/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.1.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.15
+signing_key: 
+specification_version: 4
+summary: Ruby implementations of the distributed quantile sketch algorithm DDSketch.
+test_files: []