multi_armed_bandit 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ad5425a07617c9d5751a037c73e3aa65cceb8bee
+  data.tar.gz: 1a06c8077da76fe19d9f03f2fcb2b686c416e66d
+SHA512:
+  metadata.gz: 7ec3c6c029b9838baf3456c00617cb9b536be95fe6324ab1a86b6760912e6b792ce7933ca1bddbad343f52119ae055353af7a7f015f1c3444be202e756365a9a
+  data.tar.gz: d3cba9d4509715242d1a8068701e46062661f30b30c54877be499595b6725178ddc2b723c47fad52c6331a39d988ba4d1ffbebb1a94a609a31f18a3306402a1b

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.0.0
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in multi_armed_bandit.gemspec
+gemspec

data/README.md

@@ -0,0 +1,61 @@
+# MultiArmedBandit
+
+This repo contains Ruby code for solving Multi-Armed Bandit problems. This includes the following algorithms:
+
+* Epsilon-Greedy
+* Softmax
+* Thomson Sampling with Multiple Play 
+
+Othrer major algorithms such as UCB and Bayesian Bandit will be forthcoming.
+
+## Installation
+
+By executing the following line, you can install the gem from the GitHub repo.
+
+    $ gem specific_install -l 'git://github.com/vasilyjp/multi_armed_bandit.git'
+
+
+## Usage
+
+Include MultiArmedBandit module by putting the following code.
+```ruby
+require 'multi_armed_bandit'
+include MultiArmedBandit
+```
+
+Then create an object of Softmax class. The first param is temperature. If we set temperature = 0.0, this will give us deterministic choice of the arm which has highest value. In contrast, if we set temperature = ∞, all actions have nearly the same probability. In a pracitcal sense, temperature tend to be between 0.01 and 1.0.
+
+The second param is number of arms.
+```ruby
+sm = MultiArmedBandit::Softmax.new(0.01, 3)
+```
+
+By giving lists of number of trials and rewards to bulk_update method, it returns the predicted probabilities.
+```ruby
+# Trial 1
+probs = sm.bulk_update([1000,1000,1000], [72,57,49])
+counts = probs.map{|p| (p*3000).round }
+
+# Trial 2
+probs = sm.bulk_update(counts, [154,17,32])
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/vasilyjp/multi_armed_bandit. This project is intended to be a safe, welcoming space for collaboration.
+
+
+## License
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+## Reference
+```
+[1] John Myles White: Bandit Algorithms for Website Optimization. O'Reilly Media
+[2] J. Komiyama, J. Honda, and H.Nakagawa: Optimal Regret Analysis of Thompson Sampling in Stochastic Multi-armed Bandit Problem with Multiple Plays. ICML 2015
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "multi_armed_bandit"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/multi_armed_bandit/epsilon_greedy.rb

@@ -0,0 +1,89 @@
+
+module MultiArmedBandit
+
+  class EpsilonGreedy
+
+    attr_accessor :epsilon, :counts, :values, :probs, :n_arms
+
+    # Initialize an object
+    def initialize(epsilon, n_arms)
+      @epsilon = epsilon
+      @n_arms = n_arms
+      reset()    
+    end
+
+    # Reset instance variables
+    def reset()
+      @counts = Array.new(@n_arms, 0)
+      @values = Array.new(@n_arms, 0.0)
+      @probs = Array.new(@n_arms, 0.0)
+    end
+
+    # Update in a lump. new_counts is a list of each arm's trial number and
+    # new_rewards means a list of rewards.
+    def bulk_update(new_counts, new_rewards)
+      
+      # update the numbers of each arm's trial
+      @counts = new_counts
+
+      # update expectations of each arm
+      new_values = []
+      @counts.zip( new_rewards ).each do |n, r|
+        new_values << r / n.to_f
+      end
+      @values = new_values
+
+      # calcurate probabilities
+      j = ind_max(@values)
+      for i in 0..@n_arms-1 do
+        if i == j
+          @probs[i] = 1-@epsilon
+        else
+          @probs[i] = (@epsilon)/(@n_arms-1)
+        end
+      end
+        
+      return @probs
+    end
+    
+    def update(chosen_arm, reward)
+      @counts[chosen_arm] = @counts[chosen_arm] + 1
+      n = @counts[chosen_arm]
+      
+      value = @values[chosen_arm]
+      new_value = ((n - 1) / n.to_f) * value + (1 / n.to_f) * reward
+      @values[chosen_arm] = new_value
+      return
+    end
+
+    
+    def select_arm
+      if rand > @epsilon
+        return ind_max(@values)
+      else
+        return rand(@values.size)
+      end
+    end
+
+    private
+    def ind_max(x)
+      m = x.max
+      return x.index(m)
+    end
+      
+    def categorical_draw(probs)
+      z = rand()
+      cum_prob = 0.0
+
+      probs.size().times do |i|
+        prob = probs[i]
+        cum_prob += prob
+        if cum_prob > z
+          return i
+        end
+      end
+      
+      return probs.size() - 1
+    end
+  end
+end

data/lib/multi_armed_bandit/mp_ts.rb

@@ -0,0 +1,46 @@
+require 'simple-random'
+
+module MultiArmedBandit
+
+  class MultiplePlayTS
+
+    attr_accessor :k, :l, :alpha, :beta, :arm_ids
+
+    # k: num of arms
+    # l: num of selected arms
+    def initialize(k, l, setseed=TRUE)
+      @k = k
+      @l = l
+      @r = SimpleRandom.new
+      # By default the same random seed is used, so we change it
+      @r.set_seed if setseed==TRUE
+      reset
+    end
+
+    def reset
+      @alpha = Array.new(@k, 1)
+      @beta = Array.new(@k, 1)
+      @arm_ids = Array.new(@k, '')
+    end
+
+    # Get selected arm ids
+    def get_selected_arms
+      selected_arms = @alpha.zip(@beta).zip(@arm_ids)
+                      .map{|c,i| [i, @r.beta(c[0],c[1])]}
+                      .sort_by{|v| -v[1]}
+                      .map{|v| v[0]}[0..@l-1]
+    end
+
+    # selected_arms: List of selected drawn arms
+    def update_params_draw(selected_arms)
+      selected_arms.map{|i| @beta[i]+=1}
+    end
+
+    # idx: Index number of rewarded arm
+    def update_params_reward(idx)
+      @alpha[idx]+=1
+      @beta[idx]-=1
+    end
+    
+  end
+end

data/lib/multi_armed_bandit/softmax.rb

@@ -0,0 +1,80 @@
+
+
+module MultiArmedBandit
+
+  class Softmax
+
+    attr_accessor :temperature, :counts, :values, :probs, :n_arms
+
+    # Initialize an object
+    def initialize(temperature, n_arms)
+      @n_arms = n_arms
+      @temperature = temperature
+      reset()    
+    end
+
+    # Reset instance variables
+    def reset()
+      @counts = Array.new(@n_arms, 0)
+      @values = Array.new(@n_arms, 0.0)
+      @probs = Array.new(@n_arms, 0.0)
+    end
+
+    # Update in a lump. new_counts is a list of each arm's trial number and
+    # new_rewards means a list of rewards.
+    # both each num in new_counts and new_rewards should be accumulated numbers
+    def bulk_update(new_counts, new_rewards)
+
+      # update the numbers of each arm's trial
+      @counts = new_counts
+
+      # update expectations of each arm
+      new_values = []
+      @counts.zip( new_rewards ).each do |n, reward|
+        new_values << reward / n.to_f
+      end
+      @values = new_values
+
+      # calcurate probabilities
+      z = @values.collect{|i| Math.exp(i/@temperature)}.reduce(:+)
+      @probs = @values.collect{|i| Math.exp(i/@temperature)/z}
+
+      return probs
+    end
+
+    
+    def update(chosen_arm, reward)
+      @counts[chosen_arm] = @counts[chosen_arm] + 1
+      n = @counts[chosen_arm]
+      
+      value = @values[chosen_arm]
+      new_value = ((n - 1) / n.to_f) * value + (1 / n.to_f) * reward
+      @values[chosen_arm] = new_value
+      return
+    end
+      
+    
+    def select_arm
+      z = @values.collect{|i| Math.exp(i/@temperature)}.reduce(:+)
+      @probs = @values.collect{|i| Map.exp(i/@temperature)/z}
+      return categorical_draw(@probs)
+    end
+
+    private
+    def categorical_draw(probs)
+      z = rand()
+      cum_prob = 0.0
+
+      probs.size().times do |i|
+        prob = probs[i]
+        cum_prob += prob
+        if cum_prob > z
+          return i
+        end
+      end
+      
+      return probs.size() - 1
+    end
+    
+  end
+end

data/lib/multi_armed_bandit/version.rb

@@ -0,0 +1,3 @@
+module MultiArmedBandit
+  VERSION = "0.2.0"
+end

data/lib/multi_armed_bandit.rb

@@ -0,0 +1,9 @@
+require "multi_armed_bandit/version"
+require "multi_armed_bandit/softmax"
+require "multi_armed_bandit/epsilon_greedy"
+require "multi_armed_bandit/mp_ts"
+
+module MultiArmedBandit
+  # Your code goes here...
+
+end

data/multi_armed_bandit.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'multi_armed_bandit/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "multi_armed_bandit"
+  spec.version       = MultiArmedBandit::VERSION
+  spec.authors       = ["kndt84"]
+  spec.email         = ["takashi.kaneda@vasily.jp"]
+
+  spec.summary       = %q{multi-armed bandit algorithms}
+#  spec.description   = %q{TODO: Write a longer description or delete this line.}
+  spec.homepage      = "https://github.com/vasilyjp/multi_armed_bandit"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_dependency "simple-random"
+end

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: multi_armed_bandit
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- kndt84
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simple-random
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- takashi.kaneda@vasily.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/multi_armed_bandit.rb
+- lib/multi_armed_bandit/epsilon_greedy.rb
+- lib/multi_armed_bandit/mp_ts.rb
+- lib/multi_armed_bandit/softmax.rb
+- lib/multi_armed_bandit/version.rb
+- multi_armed_bandit.gemspec
+homepage: https://github.com/vasilyjp/multi_armed_bandit
+licenses:
+- MIT
+metadata:
+  allowed_push_host: 'TODO: Set to ''http://mygemserver.com'''
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: multi-armed bandit algorithms
+test_files: []