metricstore 0.0.1 → 0.1.0

data/lib/metricstore/couchbase_client.rb

@@ -1,16 +1,70 @@
+require 'couchbase'
+
 module Metricstore
+  # Internal class. Use this class outside the gem at your own risk.
   class CouchbaseClient
-    include BaseClient
     
     def initialize(*args, &callback)
       super
+      @connection = Couchbase.connect(*args, &callback)
+    end
+
+    # key: a string
+    # delta: an integer
+    # options:
+    #  :ttl => Time-to-live (number of seconds from now).
+    # returns: [value, cas_version_id]
+    def increment(key, delta, opts={})
+      options = {:initial => delta, :extended => true}
+      options.merge(:ttl => convert_ttl(opts[:ttl])) if opts.include?(:ttl)
+      value, flags, cas = connection.incr(key, delta, options)
+      [value, cas]
+    end
+
+    # key: a string
+    # value: a marshalable object
+    # options:
+    #  :ttl => Time-to-live (number of seconds from now).
+    # returns: cas_version_id, or nil if the key already exists. 
+    def add(key, value, opts={})
+      options = {}
+      options.merge(:ttl => convert_ttl(opts[:ttl])) if opts.include?(:ttl)
+      connection.add(key, value, opts)
+    rescue Couchbase::Error::KeyExists => e
+      nil
+    end
 
-      require 'couchbase'
-      @couchbase = Couchbase.connect(*args, &callback)
+    # key: a string
+    # value: a marshalable object
+    # options:
+    #  :ttl => Time-to-live (number of seconds from now).
+    #  :cas => a version id (for optimistic concurrency control)
+    # returns: cas_version_id, or nil if the key already exists. 
+    def set(key, value, opts={})
+      options = {}
+      options.merge(:ttl => convert_ttl(opts[:ttl])) if opts.include?(:ttl)
+      options.merge(:cas => opts[:cas]) if opts.include?(:cas)
+      connection.set(key, value, opts)
+    rescue Couchbase::Error::KeyExists => e
+      nil
+    end
+
+    # key: a string
+    # returns: [value, cas_version_id], or nil if the key doesn't exist.
+    def fetch(key, opts={})
+      options = {:extended => true, :quiet => true}
+      options.merge(:ttl => convert_ttl(opts[:ttl])) if opts.include?(:ttl)
+      value, flags, cas = connection.get(key, options)
+      value.nil? ? nil : [value, cas]
     end
 
     private
 
-    attr_reader :couchbase
+    attr_reader :connection
+
+    def convert_ttl(ttl)
+      ttl.nil? ? nil : ttl.to_f <= 0 ? nil : (Time.now + ttl.to_f).to_f
+    end
+
   end
 end

data/lib/metricstore/count_incrementer.rb

@@ -0,0 +1,23 @@
+module Metricstore
+  class CountIncrementer < Updater
+
+    def increment(key, delta, ttl=nil)
+      return if delta.zero?
+      update(key, delta, ttl)
+    end
+
+    protected
+
+    def prepare_data(delta)
+      delta
+    end
+    
+    def consolidate_data(delta1, delta2)
+      delta1 + delta2
+    end
+
+    def handle_update(key, delta, ttl, errors)
+      kvstore.increment(key, delta, :ttl => ttl)
+    end
+  end
+end

data/lib/metricstore/exceptions.rb

@@ -0,0 +1,4 @@
+module Metricstore
+  class DataLossError < RuntimeError
+  end
+end

data/lib/metricstore/hyper_log_log.rb

@@ -0,0 +1,91 @@
+module Metricstore
+  module HyperLogLog
+
+    HASH_BIT_SIZE = 512
+    
+    ALPHA = {}
+    ALPHA[16] = 0.673 # m = 2**4
+    ALPHA[32] = 0.697 # m = 2**5
+    ALPHA[64] = 0.709 # m = 2**6
+    (7 .. 16).each do |b|
+      m = 2 ** b
+      ALPHA[m] = 0.7213/(1 + 1.1079/m)
+    end
+
+    class Builder
+      require 'digest/sha2'
+
+      attr_reader :bucket_count
+      
+      # bucket_updater must have a method named "call" which takes two arguments
+      #   the bucket index, and an integer value (of which it will track the max
+      #   value per bucket).
+      def initialize(error_rate, bucket_updater)
+        @error_rate = error_rate
+        unless @error_rate > 0 && @error_rate < 1
+          raise(ArgumentError, "error_rate must be between 0 and 1")
+        end
+        @bits = HyperLogLog.bits_needed(error_rate)
+        unless (@bits + 10) <= HASH_BIT_SIZE
+          raise(ArgumentError, "error_rate is unattainable. be less picky.")
+        end
+        @bucket_count = 1 << @bits
+        @alpha = ALPHA[@bucket_count]
+        @bucket_updater = bucket_updater
+      end
+
+      def add(item)
+        hashed = hash_of(item)
+        offset = HASH_BIT_SIZE - @bits
+        mask = ((1 << @bits) - 1) << offset
+        shifted_front_bits = (hashed & mask)
+        front_bits = shifted_front_bits >> offset
+        back_bits = hashed - shifted_front_bits
+        bucket_index = front_bits
+        raise("BUG!") if bucket_index >= @bucket_count
+        next_on_bit_index = (HASH_BIT_SIZE - @bits).times.find{|i| back_bits[HASH_BIT_SIZE - @bits - i] == 1}
+        if next_on_bit_index.nil?
+          next_on_bit_index= HASH_BIT_SIZE - @bits
+        else
+          next_on_bit_index += 1
+        end
+        @bucket_updater.call(bucket_index, next_on_bit_index)
+      end
+
+      private
+
+      def hash_of(item)
+        sha = Digest::SHA2.new(HASH_BIT_SIZE)
+        sha << item.to_s
+        sha.to_s.to_i(16)
+      end
+    end
+
+    def self.bits_needed(error_rate)
+      Math.log((1.04 / error_rate) ** 2, 2).round
+    end
+
+    def self.estimate_cardinality(buckets)
+      values = buckets.to_a
+      m = values.size
+      raise("BUG!") unless m > 0
+      alpha = ALPHA[m]
+      raw = alpha * (m ** 2) / values.map{|x| 2 ** -(x || 0)}.inject(:+)
+      if raw <= 2.5 * m
+        # correct for being below ideal range
+        zero_registers = values.count(nil)
+        if zero_registers == 0
+          raw
+        else
+          m * Math.log(m.to_f / zero_registers)
+        end
+      elsif raw <= (2 ** HASH_BIT_SIZE) / 30.0
+        # ideal range
+        raw
+      else
+        # correct for being beyond ideal range
+        (-2 ** HASH_BIT_SIZE) * Math.log(1 - raw.to_f/(2**HASH_BIT_SIZE), 2)
+      end
+    end
+  end
+end

data/lib/metricstore/incrementer.rb

@@ -0,0 +1,41 @@
+module Metricstore
+  class Incrementer < Updater
+
+    def increment(key, delta, ttl=nil)
+      return if delta.zero?
+      update(key, delta, ttl)
+    end
+
+    protected
+
+    def prepare_data(delta)
+      delta
+    end
+    
+    def consolidate_data(delta1, delta2)
+      delta1 + delta2
+    end
+
+    def handle_update(key, delta, ttl, errors)
+      stored_value, cas = kvstore.fetch(key, :ttl => ttl)
+      if stored_value.nil?
+        if kvstore.add(key, delta, :ttl => ttl)
+          return delta
+        else
+          # collision
+          retry_update(key, delta, ttl, errors)
+          return nil
+        end
+      else
+        new_value = stored_value + delta
+        if kvstore.set(key, new_value, :ttl => ttl, :cas => cas)
+          return new_value
+        else
+          # collision
+          retry_update(key, min_max, ttl, errors)
+          return nil
+        end
+      end
+    end
+  end
+end

data/lib/metricstore/inserter.rb

@@ -0,0 +1,78 @@
+require 'set'
+
+module Metricstore
+  class Inserter < Updater
+
+    def insert(key, values, ttl=nil)
+      update(key, values, ttl)
+    end
+
+    def list_threshold=(threshold)
+      @list_threshold = threshold
+    end
+
+    def list_threshold
+      @list_threshold ||= 100
+    end
+
+    protected
+
+    def prepare_data(values)
+      Set.new(Array(values))
+    end
+
+    def consolidate_data(values1, values2)
+      return 'overflow' if values1 == 'overflow' || values2 == 'overflow'
+      consolidated = values1 + values2
+      return 'overflow' if consolidated.size > list_threshold
+      consolidated
+    end
+
+    # Returns a list of the values that were newly inserted, or else nil
+    # if there was contention, and we have to retry.
+    def handle_update(key, values, ttl, errors)
+      return [] if values.nil? || values.empty?
+      #TODO: there's room here for a local cache optimization
+      list, cas = kvstore.fetch(key, :ttl => ttl)
+      if list.nil?
+        if values == 'overflow' || values.size > list_threshold
+          if kvstore.add(key, 'overflow', :ttl => ttl)
+            return []
+          else
+            # collision
+            retry_update(key, 'overflow', ttl, errors)
+            return nil
+          end
+        elsif kvstore.add(key, values.to_a, :ttl => ttl)
+          return values
+        else
+          # collision
+          retry_update(key, values, ttl, errors)
+          return nil
+        end
+      elsif list == 'overflow'
+        return []
+      else
+        list = Set.new(list)
+        values = values.reject{ |v| list.include?(v) }
+        return [] if values.empty?
+        new_list = values + list.to_a
+        if new_list.size > list_threshold
+          if kvstore.set(key, 'overflow', :cas => cas, :ttl => ttl)
+            return []
+          else
+            # collision
+            retry_update(key, 'overflow', ttl, errors)
+            return nil
+          end
+        elsif kvstore.set(key, new_list, :cas => cas, :ttl => ttl)
+          return values
+        else
+          # collision
+          retry_update(key, values, ttl, errors)
+          return nil
+        end
+      end
+    end
+  end
+end 

data/lib/metricstore/mock_key_value_client.rb

@@ -0,0 +1,39 @@
+module Metricstore
+  # Internal class. Use this class outside the gem at your own risk.
+  # TTL is ignored. Not thread-safe. For testing purposes only.
+  class MockKeyValueClient
+    
+    def initialize(*args, &callback)
+      @store = {}
+    end
+
+    def increment(key, delta, opts={})
+      if @store.include?(key)
+        @store[key] += delta
+      else
+        @store[key] = delta
+      end
+      [@store[key], @store[key]]
+    end
+
+    def add(key, value, opts={})
+      return nil if @store.include?(key)
+      @store[key] = value
+      [value, value]
+    end
+
+    def set(key, value, opts={})
+      return nil if opts[:cas] && opts[:cas] != @store[key]
+      @store[key] = value
+    end
+
+    def fetch(key, opts={})
+      value = @store[key]
+      value.nil? ? nil : [value, value]
+    end
+
+    def to_s
+      "MockKeyValueClient: #{@store.inspect}"
+    end
+  end
+end

data/lib/metricstore/monkey_patches.rb

@@ -0,0 +1,27 @@
+module Metricstore
+  module AllCombinations
+    def all_combinations
+      if block_given?
+        a = self.to_a
+        0.upto(size) do |n|
+          a.combination(n) do |c|
+            yield c
+          end
+        end
+      else
+        Enumerator.new do |yielder|
+          a = self.to_a
+          0.upto(size) do |n|
+            a.combination(n) do |c|
+              yielder << c
+            end
+          end
+        end
+      end
+    end
+  end
+end
+
+class Array
+  include Metricstore::AllCombinations
+end

data/lib/metricstore/range_updater.rb

@@ -0,0 +1,58 @@
+module Metricstore
+  class RangeUpdater < Updater
+
+    def update_range(key, value, ttl=nil)
+      raise(ArgumentError, "value must be numeric") unless value.is_a?(Numeric)
+      update(key, [value, value], ttl)
+    end
+
+    protected
+
+    def prepare_data(min_max)
+      min_max
+    end
+
+    def consolidate_data(min_max1, min_max2)
+      [min(min_max1[0], min_max2[0]), max(min_max1[1], min_max2[1])]
+    end
+
+    # Returns nil if there was contention, and we have to retry.
+    # Returns [:new, range] where range is (max - min), if range was added.
+    # Otherwise returns [:grew, diff] where diff is the amount the range grew.
+    def handle_update(key, min_max, ttl, errors)
+      #TODO: there's room here for a local cache optimization
+      stored_min_max, cas = kvstore.fetch(key, :ttl => ttl)
+      if stored_min_max.nil?
+        if kvstore.add(key, min_max, :ttl => ttl)
+          return [:new, (min_max[1] - min_max[0])]
+        else
+          # collision
+          retry_update(key, min_max, ttl, errors)
+          return nil
+        end
+      else
+        stored_min, stored_max = stored_min_max
+        new_min = min(stored_min, min_max[0])
+        new_max = max(stored_max, min_max[1])
+        return 0 if new_min == stored_min && new_max == stored_max
+        if kvstore.set(key, [new_min, new_max], :ttl => ttl, :cas => cas)
+          return [:grew, (stored_min - new_min) + (new_max - stored_max)]
+        else
+          # collision
+          retry_update(key, min_max, ttl, errors)
+          return nil
+        end
+      end
+    end
+
+    private
+
+    def min(a, b)
+      a.nil? ? b : b.nil? ? nil : (a < b) ? a : b
+    end
+
+    def max(a, b)
+      a.nil? ? b : b.nil? ? nil : (a < b) ? b : a
+    end
+  end
+end

data/lib/metricstore/updater.rb

@@ -0,0 +1,192 @@
+require 'eventmachine'
+
+module Metricstore
+
+  # Abstract class. Not thread-safe.
+  #
+  # Sub-classes must implement (protected) methods:
+  #
+  #   prepare_data(data)
+  #
+  #   consolidate_data(data1, data2)
+  #
+  #   handle_update(key, data, ttl, errors)
+  #    -> must return a truthy value if and only if the update occurred.
+  #
+  class Updater
+
+    # opts:
+    #   :sleep_interval - sleep cycle length in seconds (default: 0.1).
+    #   :kvstore  - the underlying key-value store.
+    #   :max_retry_delay_in_seconds - maximum length of time to wait after an error.
+    #   :max_unhandled_errors - maximum number of retries before handling errors. Set this >= max_healthy_errors.
+    #   :max_healthy_errors - maximum number of retries before healhty? returns false. Set this <= max_unhandled_errors.
+    def initialize(opts={})
+      @sleep_interval = (opts[:sleep_interval] || 0.1).to_f
+      @kvstore = required(opts, :kvstore)
+      @max_retry_delay = required(opts, :max_retry_delay_in_seconds).to_f
+      @max_unhandled_errors = required(opts, :max_unhandled_errors).to_i
+      @max_healthy_errors = required(opts, :max_healthy_errors).to_i
+
+      @timer = nil
+      @running = false
+      @healthy = nil
+      @pending_updates = {}
+    end
+
+    def start!
+      return if @running
+      @running = true
+      EM.next_tick { process! }
+    end
+
+    # Be sure to call this after tests, when you want to let go of the object.
+    def stop!
+      @running = false
+      if timer = @timer
+        EM.cancel_timer(timer)
+        @timer = nil
+      end
+    end
+
+    def healthy?
+      @healthy != false
+    end
+
+    # Approximate length of the queue
+    def backlog
+      @pending_updates.size
+    end
+
+    attr_accessor :handle_update_result
+
+    protected
+
+    attr_reader :kvstore
+
+    def required(opts, parameter_name)
+      opts[parameter_name] || raise("Missing parameter: #{parameter_name}")
+    end
+
+    def retry_update(key, data, ttl=nil, errors=[])
+      update(key, data, ttl, errors)
+    end
+
+    def update(key, data, ttl=nil, errors=[])
+      schedule(errors) do
+        pend_update(key, data, ttl, errors)
+      end
+    end
+
+    def consolidate_data(data1, data2)
+      raise NotImplementedError
+    end
+
+    def prepare_data(data)
+      raise NotImplementedError
+    end
+
+    # Sub-classes may want to over-ride this. 
+    def handle_error(e)
+      if defined?(Airbrake) && Airbrake.configuration.environment_name && Airbrake.configuration.public?
+        Airbrake.notify(e)
+      elsif defined?(Exceptional) && Exceptional::Config.should_send_to_api?
+        Exceptional.handle(e)
+      else
+        puts e.inspect
+        raise
+      end
+    end
+
+    private
+
+    attr_reader :sleep_interval
+    attr_reader :max_healthy_errors
+    attr_reader :max_retry_delay
+    attr_reader :max_unhandled_errors
+
+    def schedule(errors=[], &callback)
+      if errors.size == 0
+        EM.next_tick { callback.call }
+      else
+        EM.add_timer(retry_delay_for(errors)) { callback.call }
+      end
+    end
+
+    def retry_delay_for(errors)
+      [2 ** (errors.size - 4), max_retry_delay / 2.0].min * (1 + rand)
+    end
+
+    # This must only be called by the EM reactor thread
+    def pend_update(key, data, ttl, errors)
+      pending = @pending_updates[key]
+      if pending.nil?
+        @pending_updates[key] = {:data => prepare_data(data), :ttl => ttl, :errors => errors}
+      else
+        pending[:data]    = consolidate_data(pending[:data], prepare_data(data))
+        pending[:ttl]     = max_ttl(ttl, pending[:ttl])
+        pending[:errors] += errors
+      end
+    rescue => e
+      handle_error(e)
+    end
+
+    def max_ttl(a, b)
+      return 0 if a.nil? || b.nil? || a == 0 || b == 0
+      [a,b].max
+    end
+
+    def reschedule_process!(sleep_first)
+      if @running
+        if timer = @timer
+          EM.cancel_timer(timer)
+        end
+        if sleep_first
+          @timer = EM.add_timer(sleep_interval) { process! }
+        else
+          EM.next_tick { process! }
+        end
+      end
+    end
+
+    def process!
+      @timer = nil
+      processed = 0
+      until @pending_updates.empty? || (processed+=1) > 10
+        key, update = @pending_updates.shift
+        process_update(key, update[:data], update[:ttl], update[:errors] || [])
+      end
+      reschedule_process!(@pending_updates.empty?)
+    rescue => e
+      handle_error(e)
+    end
+
+    def process_update(key, data, ttl, errors)
+      result = handle_update(key, data, ttl, errors)
+      unless result.nil?
+        @healthy = true
+        handle_update_result.call(key, result, ttl) if handle_update_result
+      end
+    rescue => e
+      # Uh oh. We stick the update back in the queue before handling the error.
+      begin
+        errors << e
+        @healthy = false if errors.size > max_healthy_errors
+        if errors.size <= max_unhandled_errors
+          update(key, data, ttl, errors)
+        else
+          update(key, data, ttl, [])
+          handle_each_error(errors)
+        end
+      rescue => e2
+        handle_error(e2) # bugs on bugs on bugs if you get here!
+      end
+    end
+
+    def handle_each_error(errors)
+      errors.uniq{|error| [error.message, error.backtrace]}.each do |error|
+        handle_error(error)
+      end
+    end
+  end
+end