forget_table 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 51b34b5b1d37ac5d704a4d7e48c6b635fb8de7b0
+  data.tar.gz: 276ca5952d1e6bdcca27c1c257c8b1fb4b3054d7
+SHA512:
+  metadata.gz: d28a09e53676e1b908a8bdcefded130c241b71bdebf5610c48d0e2b25c39d063293f1efc1c82483c8c7552630efc3def2844a103fa98ed0833bb924ad9b03129
+  data.tar.gz: 8abb4e1332f2aeac8f2423d09b47fdfd081ae8d86c853915b503662b9d2d9687e3da7865d753faf46dbe89304784dc0a9c1797aad5f7973a901f6058ce54c5cf

data/lib/forget_table/decay.rb

@@ -0,0 +1,44 @@
+module ForgetTable
+
+  class Decay
+
+    DEFAULT_DECAY_RATE = 0.05
+
+    # - last_updated: timestamp of the last update
+    # - rate:         the rate of the decay (optional)
+    def initialize(last_updated, rate = DEFAULT_DECAY_RATE)
+      @last_updated = last_updated
+      @rate = rate
+    end
+
+    def decay_value(value)
+      decayed_value = value - decay_for(value)
+      decayed_value > 0 ? decayed_value : 1
+    end
+
+    private
+
+    attr_reader :last_updated, :rate
+
+    def decay_for(value)
+      poisson(decay_factor * value)
+    end
+
+    def decay_factor
+      rate * tau
+    end
+
+    # Time since last update
+    def tau
+      [current_time - last_updated, 1].max
+    end
+
+    def poisson(value)
+      Poisson.new(value).sample
+    end
+
+    def current_time
+      @timestamp ||= Time.now.to_i
+    end
+  end
+end

data/lib/forget_table/decrementer.rb

@@ -0,0 +1,54 @@
+require 'forget_table/distribution_decrementer'
+
+module ForgetTable
+
+  class Decrementer
+
+    def initialize(redis, weighted_distribution)
+      @redis = redis
+      @weighted_distribution = weighted_distribution
+    end
+
+    def run!
+      decremented_distribution = distribution_decrementer.decremented_distribution
+      updated_redis(decremented_distribution)
+    end
+
+    private
+
+    attr_reader :redis, :weighted_distribution
+
+    # Updates:
+    # 1. the weighted distribution with the new distribution
+    # 2. the total number of hits for the distribution with the new count
+    # 3. the last_updated_at value with the current time
+
+    def updated_redis(distribution)
+      redis.pipelined do
+        redis.zadd(distribution.name, distribution.bins.to_a.map(&:reverse))
+        redis.set(hits_count_key, distribution.hits_count)
+        redis.set(last_updated_at_key, Time.now.to_i)
+      end
+    end
+
+    def distribution_decrementer
+      last_updated_at = Integer(@redis.get(last_updated_at_key))
+      distribution_decrementer = DistributionDecrementer.new(
+        weighted_distribution: weighted_distribution,
+        last_updated_at: last_updated_at
+      )
+    end
+
+    def distribution_keys
+      DistributionKeys.new(weighted_distribution.name)
+    end
+
+    def hits_count_key
+      distribution_keys.hits_count
+    end
+
+    def last_updated_at_key
+      distribution_keys.last_updated_at
+    end
+  end
+end

data/lib/forget_table/distribution.rb

@@ -0,0 +1,104 @@
+require 'forget_table/decrementer'
+require 'forget_table/weighted_distribution'
+require 'forget_table/distribution_keys'
+
+module ForgetTable
+
+  # Represents a categorical distribution composed by weighted bins.
+  #
+  # A distribution is instantiated with the following parameters:
+  # - name: the name of the distribution
+  # - redis: the redis client that will host the distribution
+  #
+  # Example of an instance:
+  #   distribution: "guitars"
+  #     bins: "fender" => 10, "gibson" => 20, "epi" => "30
+
+  class Distribution
+    attr_reader :name
+
+    def initialize(name:, redis:)
+      @name = name
+      @redis = redis
+    end
+
+    # Increments the bin score by the given amount.
+    # params:
+    # - bin
+    # - amount
+    def increment(bin:, amount: 1)
+      redis.zincrby(name, amount, bin)
+
+      # Increment the total number of hits
+      stored_bins = redis.incrby(hits_count_key, 1)
+
+      if stored_bins == 1
+        # Set the initial timestamp if never set
+        redis.set(last_updated_at_key, Time.now.to_i)
+      end
+    end
+
+    # Returns the list of bins in the distribution.
+    # Params:
+    # - number_of_bins
+    # - options
+    def distribution(number_of_bins: -1, with_scores: false)
+      begin
+        decrement!
+
+        stop_bin = (number_of_bins == -1) ? -1 : (number_of_bins - 1)
+        redis.zrevrange(name, 0, stop_bin, with_scores: with_scores)
+      rescue RuntimeError
+        [[]]
+      end
+    end
+
+    # Returns the score for the given bin
+    def score_for_bin(bin)
+      decrement!
+
+      redis.zscore(name, bin)
+    end
+
+    def last_updated
+      redis.get(last_updated_at_key)
+    end
+
+    def hits_count
+      redis.get(hits_count_key).to_i
+    end
+
+    private
+    attr_reader :redis
+
+    def hits_count_key
+      distribution_keys.hits_count
+    end
+
+    def last_updated_at_key
+      distribution_keys.last_updated_at
+    end
+
+    def decrement!
+      raise "Cannot find distribution #{name}" unless redis.exists(name)
+
+      decrementer.run!
+    end
+
+    def decrementer
+      Decrementer.new(redis, weighted_distribution)
+    end
+
+    def weighted_distribution
+      bins = Hash[*redis.zrevrange(name, 0, -1, with_scores: true).flatten]
+      WeightedDistribution.new(
+        name: name,
+        bins: bins,
+      )
+    end
+
+    def distribution_keys
+      DistributionKeys.new(name)
+    end
+  end
+end

data/lib/forget_table/distribution_decrementer.rb

@@ -0,0 +1,32 @@
+module ForgetTable
+
+  class DistributionDecrementer
+
+    def initialize(weighted_distribution:, last_updated_at:)
+      @weighted_distribution = weighted_distribution
+      @last_updated_at = last_updated_at
+    end
+
+    def decremented_distribution
+      WeightedDistribution.new(
+        name: weighted_distribution.name,
+        bins: decremented_bins
+      )
+    end
+
+    private
+
+    attr_reader :weighted_distribution, :last_updated_at
+
+    def decremented_bins
+      decremented_values = decrement(weighted_distribution.values)
+      bin_names = weighted_distribution.bin_names
+      Hash[*bin_names.zip(decremented_values).flatten]
+    end
+
+    def decrement(values)
+      decay = Decay.new(last_updated_at)
+      values.map { |value| decay.decay_value(value) }
+    end
+  end
+end

data/lib/forget_table/distribution_keys.rb

@@ -0,0 +1,18 @@
+module ForgetTable
+
+  class DistributionKeys
+    attr_reader :distribution_name
+
+    def initialize(distribution_name)
+      @distribution_name = distribution_name
+    end
+
+    def last_updated_at
+      "#{distribution_name}/last_updated_at"
+    end
+
+    def hits_count
+      "#{distribution_name}/hits_count"
+    end
+  end
+end

data/lib/forget_table/poisson.rb

@@ -0,0 +1,40 @@
+module ForgetTable
+
+  class Poisson
+    attr_reader :average
+
+    def initialize(average)
+      raise ArgumentError, "average must be > 0 , #{average} given" if average <= 0
+      @average = average
+    end
+
+    def sample
+      @sample ||= extract_sample
+    end
+
+    private
+
+    # Returns an Integer extracted from a Poisson
+    # distribution with average `average`.
+    # Implemented according to the Knuth algorithm.
+    def extract_sample
+      l = Math.exp(-average)
+      k = 0
+      p = 1
+      while p > l do
+        k += 1
+        p *= random_in_0_1
+      end
+      k - 1
+    end
+
+    def random_in_0_1
+      random.rand(1.0)
+    end
+
+    def random
+      @@rand ||= Random.new
+    end
+  end
+end
+

data/lib/forget_table/weighted_distribution.rb

@@ -0,0 +1,23 @@
+module ForgetTable
+
+  class WeightedDistribution
+    attr_reader :name, :bins
+
+    def initialize(name:, bins:)
+      @name = name
+      @bins = bins
+    end
+
+    def values
+      bins.values
+    end
+
+    def bin_names
+      bins.keys
+    end
+
+    def hits_count
+      values.inject(:+).to_i
+    end
+  end
+end

data/lib/forget_table.rb

@@ -0,0 +1,9 @@
+require 'redis'
+
+require 'forget_table/decay'
+require 'forget_table/decrementer'
+require 'forget_table/distribution'
+require 'forget_table/distribution_decrementer'
+require 'forget_table/distribution_keys'
+require 'forget_table/poisson'
+require 'forget_table/weighted_distribution'

data/spec/forget_table/decay_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe ForgetTable::Decay do
+
+  class FakePoisson
+    attr_reader :sample
+    def initialize(sample)
+      @sample = sample
+    end
+  end
+
+  let(:last_updated) { 1_000 }
+
+  before do
+    allow(ForgetTable::Poisson).to receive(:new) { |arg| FakePoisson.new(arg) }
+  end
+
+  describe "#decay" do
+    let(:decay) { ForgetTable::Decay.new(last_updated, 0.01) }
+
+    it "returns new decayed value" do
+      set_current_time(1_010)
+
+      expect(decay.decay_value(10)).to eq(9)
+    end
+
+    it "replaces negative decayed values with 1" do
+      set_current_time(10_000)
+
+      expect(decay.decay_value(20)).to eq(1)
+    end
+
+    def set_current_time(current_time)
+      allow(Time).to receive_message_chain(:now, :to_i).and_return(current_time)
+    end
+  end
+end

data/spec/forget_table/decrementer_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+require 'fakeredis'
+
+describe ForgetTable::Decrementer do
+
+  let(:redis) { Redis.new(port: 10000) }
+  let(:distribution_name) { "guitars" }
+  let(:distribution_keys) { ForgetTable::DistributionKeys.new("guitars") }
+
+  let(:weighted_distribution) do
+    double(:weighted_distribution, name: distribution_name)
+  end
+
+  let(:distribution_decrementer) do
+    double(:distribution_decrementer, decremented_distribution: decremented_distribution)
+  end
+
+  let(:decremented_distribution) do
+    double(:decremented_distribution,
+           name: distribution_name,
+           bins: { "fender" => 10.0, "gibson" => 20.0 },
+           hits_count: 30.0,
+          )
+  end
+
+  let(:decrementer) do
+    ForgetTable::Decrementer.new(
+      redis,
+      weighted_distribution
+    )
+  end
+
+  before(:each) do
+    redis.flushall
+    redis.set(distribution_keys.last_updated_at, 123)
+
+    allow(ForgetTable::DistributionDecrementer).to receive(:new).
+      with(
+        weighted_distribution: weighted_distribution,
+        last_updated_at: 123
+    ) { distribution_decrementer }
+  end
+
+  describe "#run!" do
+    it "updates the `hits_count` key after decrementing" do
+      decrementer.run!
+
+      expect(redis.get(distribution_keys.hits_count).to_i).to eq(30)
+    end
+
+    it "updates the timestamp" do
+      allow(Time).to receive_message_chain(:now, :to_i) { 37 }
+
+      decrementer.run!
+
+      expect(redis.get(distribution_keys.last_updated_at).to_i).to eq(37)
+    end
+
+    it "stores the new values for the distribution" do
+      decrementer.run!
+
+      distribution = redis.zrevrange(distribution_name, 0, -1, with_scores: true)
+      expect(distribution).to match_array(
+        [
+          ["fender", 10.0],
+          ["gibson", 20.0],
+        ])
+    end
+  end
+end

data/spec/forget_table/distribution_decrementer_spec.rb

@@ -0,0 +1,50 @@
+require 'spec_helper'
+require 'forget_table/distribution_decrementer'
+
+describe ForgetTable::DistributionDecrementer do
+
+  let(:last_updated_at) { "updated_at" }
+  let(:weighted_distribution) do
+    double(:dist,
+           name: "foo",
+           values: [13, 17],
+           bin_names: %w(fender gibson)
+          )
+  end
+
+  let(:dist_decrementer) do
+    ForgetTable::DistributionDecrementer.new(
+      weighted_distribution: weighted_distribution,
+      last_updated_at: last_updated_at
+    )
+  end
+
+  before do
+    allow(ForgetTable::Decay).to receive(:new).with(last_updated_at) { |arg| FakeDecay.new(arg) }
+  end
+
+  class FakeDecay
+    def initialize(*); end
+
+    # Just decrement by 1
+    def decay_value(value)
+      value - 1
+    end
+  end
+
+  describe "#decremented_distribution" do
+    it "returns a new distribution with the same name" do
+      expect(dist_decrementer.decremented_distribution.name).to eq("foo")
+    end
+
+    it "returns a new distribution with decremented bins" do
+      decremented_distribution = double
+      allow(ForgetTable::WeightedDistribution).to receive(:new).with(
+        name: "foo",
+        bins: { "fender" => 12, "gibson" => 16 }
+      ).and_return(decremented_distribution)
+
+      expect(dist_decrementer.decremented_distribution).to eq(decremented_distribution)
+    end
+  end
+end

data/spec/forget_table/distribution_spec.rb

@@ -0,0 +1,89 @@
+require 'spec_helper'
+require 'fakeredis'
+
+describe ForgetTable::Distribution do
+
+  # Test with a real redis server
+  let(:redis) { Redis.new(port: 10000) }
+  let(:decrementer) { double(:decrementer) }
+  let(:distribution) { ForgetTable::Distribution.new(name: "guitars", redis: redis) }
+
+  before(:each) do
+    redis.flushall
+    allow(ForgetTable::Decrementer).to receive(:new).and_return(decrementer)
+    allow(decrementer).to receive(:run!)
+  end
+
+  describe "#name" do
+    it "reads the correct distribution_name" do
+      expect(distribution.name).to eq("guitars")
+    end
+  end
+
+  describe "#increment" do
+
+    it "insert the value if it was not existing before" do
+      distribution.increment(bin: "fender", amount: 10)
+
+      expect(distribution.distribution).to eq(["fender"])
+    end
+
+    it "sets the initial value if the item was not there before" do
+      distribution.increment(bin: "fender", amount: 10)
+
+      expect(distribution.score_for_bin("fender")).to eq(10)
+    end
+
+    it "increments the existing value" do
+      distribution.increment(bin: "fender", amount: 10)
+      distribution.increment(bin: "fender", amount: 1)
+
+      expect(distribution.score_for_bin("fender")).to eq(11)
+    end
+  end
+
+  describe "#distribution" do
+    before do
+      distribution.increment(bin: "epiphone", amount: 10)
+      distribution.increment(bin: "gibson", amount: 20)
+      distribution.increment(bin: "fender", amount: 30)
+    end
+
+    it "returns the list of all stored items" do
+      expect(distribution.distribution).to match_array(["epiphone", "gibson", "fender"])
+    end
+
+    it "returns the list of the top n stored item" do
+      expect(distribution.distribution(number_of_bins: 2)).to match_array(["gibson", "fender"])
+    end
+
+    it "returns at most the stored elements even if asked for more" do
+      expect(distribution.distribution(number_of_bins: 42)).to match_array(["epiphone", "gibson", "fender"])
+    end
+
+    it "accepts a with_scores parameter" do
+      expect(distribution.distribution(number_of_bins: 3, with_scores: true)).
+        to match_array([["epiphone", 10.0], ["gibson", 20.0], ["fender", 30.0]])
+    end
+
+    it "returns and empty hash if the distribution is not stored in redis" do
+      distribution = ForgetTable::Distribution.new(name: "foo", redis: redis)
+
+      expect(distribution.distribution).to eq([[]])
+    end
+  end
+
+  describe "#score_for_bin" do
+    it "returns the score for a given stored item" do
+      distribution.increment(bin: "ibanez", amount: 37)
+
+      expect(distribution.score_for_bin("ibanez")).to eq(37)
+    end
+
+    it "raises an exception if the distribution is not stored in redis" do
+      distribution = ForgetTable::Distribution.new(name: "foo", redis: redis)
+
+      expect{ distribution.score_for_bin("yolo") }.to raise_error
+    end
+  end
+end

data/spec/forget_table/poisson_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe ForgetTable::Poisson do
+
+  context ".sample" do
+
+    REPETITIONS = 1000
+
+    it "raises an exception if average is negative" do
+      expect { ForgetTable::Poisson.new(-1) }.to raise_error(ArgumentError)
+    end
+
+    it "should always generate an integer" do
+      REPETITIONS.times do
+        expect(ForgetTable::Poisson.new(37).sample).to be_kind_of(Integer)
+      end
+    end
+
+    it "should always be within the acceptance range" do
+      sum = 0
+      REPETITIONS.times do
+        sum += ForgetTable::Poisson.new(37).sample
+      end
+      sum /= REPETITIONS.to_f
+
+      expect(sum).to be_within(1).of(37)
+    end
+  end
+end

data/spec/forget_table/weighted_distribution_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'forget_table/weighted_distribution'
+
+describe ForgetTable::WeightedDistribution do
+
+  let(:distribution) do
+    ForgetTable::WeightedDistribution.new(
+      name: "foo",
+      bins: { "fender" => 10, "gibson" => 20 }
+    )
+  end
+
+  it "returns the distribution name" do
+    expect(distribution.name).to eq("foo")
+  end
+
+  it "returns the array of values" do
+    expect(distribution.values).to eq([10, 20])
+  end
+
+  it "returns the array of bin names" do
+    expect(distribution.bin_names).to eq(["fender", "gibson"])
+  end
+
+  it "returns the total number of hits" do
+    expect(distribution.hits_count).to eq(30)
+  end
+end

data/spec/integration/integration_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+require 'fakeredis'
+
+describe "One bin distribution" do
+
+  let(:redis) { Redis.new(port: 10000) }
+  let(:distribution) { ForgetTable::Distribution.new(name: "guitars", redis: redis) }
+
+  before do
+    redis.flushall
+  end
+
+  describe "single bin" do
+    context "with a single increment" do
+      before do
+        distribution.increment(bin: "fender", amount: 10)
+      end
+
+      it "returns the only bin" do
+        expect(distribution.distribution).to eq(["fender"])
+      end
+
+      it "returns a positive score for the bin" do
+        expect(distribution.score_for_bin("fender")).to be > 0
+      end
+
+      it "returns a positive number of hits" do
+        expect(distribution.hits_count).to be > 0
+      end
+    end
+
+    context "with multiple increment" do
+      before do
+        distribution.increment(bin: "fender", amount: 10)
+        distribution.increment(bin: "fender", amount: 20)
+        distribution.increment(bin: "fender", amount: 30)
+      end
+
+      it "returns the only bin" do
+        expect(distribution.distribution).to eq(["fender"])
+      end
+
+      it "returns a positive score for the bin" do
+        expect(distribution.score_for_bin("fender")).to be > 0
+      end
+
+      it "returns a positive number of hits" do
+        expect(distribution.hits_count).to be > 0
+      end
+    end
+  end
+
+  describe "multiple bins" do
+    context "with single increment" do
+      before do
+        distribution.increment(bin: "fender", amount: 10)
+        distribution.increment(bin: "gibson", amount: 20)
+      end
+
+      it "returns both bins" do
+        expect(distribution.distribution).to match_array(["fender", "gibson"])
+      end
+
+      it "returns a positive score for the bins" do
+        expect(distribution.score_for_bin("fender")).to be > 0
+        expect(distribution.score_for_bin("gibson")).to be > 0
+      end
+
+      it "respects the scoring order" do
+        fender_score = distribution.score_for_bin("fender")
+        gibson_score = distribution.score_for_bin("gibson")
+
+        expect(gibson_score).to be >= fender_score
+      end
+
+      it "returns a positive number of hits" do
+        expect(distribution.hits_count).to be > 0
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'forget_table'
+
+require 'byebug'

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: forget_table
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nicolò Calcavecchia
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: fakeredis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: An implementation of http://word.bitly.com/post/41284219720/forget-table
+  in ruby
+email: calcavecchia@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/forget_table.rb
+- lib/forget_table/decay.rb
+- lib/forget_table/decrementer.rb
+- lib/forget_table/distribution.rb
+- lib/forget_table/distribution_decrementer.rb
+- lib/forget_table/distribution_keys.rb
+- lib/forget_table/poisson.rb
+- lib/forget_table/weighted_distribution.rb
+- spec/forget_table/decay_spec.rb
+- spec/forget_table/decrementer_spec.rb
+- spec/forget_table/distribution_decrementer_spec.rb
+- spec/forget_table/distribution_spec.rb
+- spec/forget_table/poisson_spec.rb
+- spec/forget_table/weighted_distribution_spec.rb
+- spec/integration/integration_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/ncalca/forgettable
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Keep track of dynamically changing categorical distribution
+test_files: []
+has_rdoc: