classic_bandit 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9ce78adca73218db90a909daec3ccbb457417620d78ae2566c0a32ac7024a313
+  data.tar.gz: 9b6debd514866747242bd00de9db4d97d1fd95e5a03166e52c96b812daa3d7f6
+SHA512:
+  metadata.gz: 80cc50641016e81853f766645d1fe9baf3c7bf4b09c8bc9303ce083dac1bec574d78432dd174953785e3dd37032e34c2248d22f47acd1c08c02078f02ad6f8a7
+  data.tar.gz: e5b534ad79cc2a91a95b617c315335cb8c399c7241308750d868b4eb13e60016382765ff1e83447495676414e1213a6b7212d31e9b18f2859063d5f491f80b25

data/.devcontainer/devcontainer.json

@@ -0,0 +1,22 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ruby
+{
+	"name": "Ruby",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/ruby:1-3.3-bullseye"
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "ruby --version",
+
+	// Configure tool-specific properties.
+	// "customizations": {},
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,16 @@
+AllCops:
+  TargetRubyVersion: 3.0
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes
+
+Metrics/BlockLength:
+  Exclude:
+    - 'spec/**/*'
+
+Style/Documentation:
+  Exclude:
+    - '**/*'

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# Changelog
+
+## [0.1.0] - 2024-12-27
+
+- Initial release
+- implemented TS algorithm
+- implemented Softmax algorithm
+- implemented UCB1
+- implemented epsilon greedy

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Kohei Tsuyuki
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,142 @@
+# ClassicBandit
+
+[![CI](https://github.com/t-chov/classic_bandit/actions/workflows/ci.yml/badge.svg)](https://github.com/t-chov/classic_bandit/actions/workflows/ci.yml)
+
+A Ruby library for classic (non-contextual) multi-armed bandit algorithms including Thompson Sampling, UCB1, and Epsilon-Greedy.
+
+## Requirements
+
+- Ruby >= 3.0.0
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'classic_bandit'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install classic_bandit
+```
+
+## Usage
+
+### A/B Testing Example
+
+```ruby
+require 'classic_bandit'
+
+# Initialize banners for A/B testing
+arms = [
+  ClassicBandit::Arm.new(id: 'banner_a', name: 'Spring Campaign'),
+  ClassicBandit::Arm.new(id: 'banner_b', name: 'Summer Campaign')
+]
+
+# Choose algorithm: Epsilon-Greedy with 10% exploration
+bandit = ClassicBandit::EpsilonGreedy.new(arms: arms, epsilon: 0.1)
+
+# In your application
+selected_arm = bandit.select_arm
+# Display the selected banner to user
+show_banner(selected_arm.id)
+
+# Update with user's response
+# 1 for click, 0 for no click
+bandit.update(selected_arm, 1)
+```
+
+## Available Algorithms
+
+### Epsilon-Greedy
+
+Balances exploration and exploitation with a fixed exploration rate.
+
+```ruby
+bandit = ClassicBandit::EpsilonGreedy.new(arms: arms, epsilon: 0.1)
+```
+
+- Simple
+- Explicitly controls exploration with ε parameter
+- Explores randomly with probability ε, exploits best arm with probability 1-ε
+
+### UCB1
+
+Upper Confidence Bound algorithm that automatically balances exploration and exploitation.
+
+```ruby
+bandit = ClassicBandit::Ucb1.new(arms: arms)
+```
+
+- No explicit exploration parameter needed
+- Automatically balances exploration and exploitation
+- Uses confidence bounds to select arms
+- Always tries untested arms first
+
+### Softmax
+
+Temperature-based algorithm that selects arms according to their relative rewards.
+
+```ruby
+bandit = ClassicBandit::Softmax.new(
+  arms: arms,
+  initial_temperature: 1.0,
+  k: 0.5
+)
+```
+
+- Uses Boltzmann distribution for arm selection
+- Higher temperature leads to more exploration
+- Temperature decreases over time for better exploitation
+- Smooth probability distribution over arms
+
+### Thompson Sampling
+
+Bayesian approach that maintains a probability distribution over each arm's rewards.
+
+```ruby
+bandit = ClassicBandit::ThompsonSampling.new(arms: arms)
+```
+
+- Naturally balances exploration and exploitation
+- Uses Beta distribution to model uncertainty
+- Performs well in practice with no tuning required
+- Adapts quickly to reward patterns
+
+### Common Interface
+All algorithms share the same interface:
+
+```ruby
+# Select an arm
+arm = bandit.select_arm
+
+# Update the arm with reward
+bandit.update(arm, 1)  # Success
+bandit.update(arm, 0)  # Failure
+```
+
+## Development
+
+After checking out the repo, run:
+```bash
+$ bundle install
+$ bundle exec rspec
+```
+
+To release a new version:
+
+1. Update the version number in version.rb
+2. Create a git tag for the version
+3. Push git commits and tags
+
+### License
+
+The gem is available as open source under the terms of the MIT License.

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/classic_bandit/arm.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Represents an arm in a multi-armed bandit problem.
+  # Each arm maintains its own trial counts and success counts,
+  # which are used by various bandit algorithms to make decisions.
+  #
+  # @example Create a new arm
+  #   arm = ClassicBandit::Arm.new(id: 1, name: "banner_a")
+  #   arm.trials  #=> 0
+  #   arm.successes #=> 0
+  class Arm
+    attr_reader :id, :name
+    attr_accessor :trials, :successes
+
+    def initialize(id:, name: nil, trials: 0, successes: 0)
+      @id = id
+      @name = name || id.to_s
+      @trials = trials
+      @successes = successes
+    end
+
+    # Calculate mean reward (success rate) for this arm
+    # @return [Float] Mean reward (0.0 if no trials)
+    def mean_reward
+      return 0.0 if @trials.zero?
+
+      @successes.to_f / @trials
+    end
+  end
+end

data/lib/classic_bandit/arm_updatable.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Provides common update functionality for bandit algorithms
+  # to update arm statistics with observed rewards.
+  #
+  # @example Update an arm with a reward
+  #   class MyBandit
+  #     include ArmUpdatable
+  #   end
+  #
+  #   bandit = MyBandit.new
+  #   bandit.update(selected_arm, reward: 1)
+  module ArmUpdatable
+    # Update the selected arm with the observed reward
+    # @param arm [Arm] The arm that was selected
+    # @param reward [Integer] The observed reward (0 or 1)
+    def update(arm, reward)
+      validate_reward!(reward)
+
+      arm.trials += 1
+      arm.successes += reward
+    end
+
+    private
+
+    def validate_reward!(reward)
+      return if [0, 1].include?(reward)
+
+      raise ArgumentError, "reward must be 0 or 1"
+    end
+  end
+end

data/lib/classic_bandit/epsilon_greedy.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Implements the Epsilon-Greedy algorithm for multi-armed bandit problems.
+  # This algorithm makes a random choice with probability epsilon (exploration)
+  # and chooses the arm with the highest mean reward with probability 1-epsilon (exploitation).
+  #
+  # @example Create and use epsilon-greedy bandit
+  #   arms = [
+  #     ClassicBandit::Arm.new(id: 1, name: "banner_a", trials: 100, successes: 10),
+  #     ClassicBandit::Arm.new(id: 2, name: "banner_b", trials: 150, successes: 14)
+  #   ]
+  #   bandit = ClassicBandit::EpsilonGreedy.new(arms: arms, epsilon: 0.1)
+  #   selected_arm = bandit.select_arm
+  #   bandit.update(selected_arm, reward: 1)
+  class EpsilonGreedy
+    include ArmUpdatable
+
+    attr_reader :arms, :epsilon
+
+    def initialize(arms:, epsilon: 0.1)
+      @arms = arms
+      @epsilon = epsilon
+
+      validate_epsilon!
+    end
+
+    def select_arm
+      # If no arms have been tried, do random selection
+      return @arms.sample if @arms.all? { |arm| arm.trials.zero? }
+
+      if rand < @epsilon
+        # Exploration: random selection
+        @arms.sample
+      else
+        # Exploitation: select arm with highest mean reward
+        @arms.max_by(&:mean_reward)
+      end
+    end
+
+    private
+
+    def validate_epsilon!
+      return if (0..1).cover?(@epsilon)
+
+      raise ArgumentError, "epsilon must be between 0 and 1"
+    end
+  end
+end

data/lib/classic_bandit/softmax.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Implements the Softmax algorithm for multi-armed bandit problems.
+  # This algorithm selects arms based on Boltzmann distribution,
+  # with temperature parameter controlling exploration-exploitation balance.
+  #
+  # @example Create and use Softmax bandit
+  #   arms = [
+  #     ClassicBandit::Arm.new(id: 1, name: "banner_a"),
+  #     ClassicBandit::Arm.new(id: 2, name: "banner_b")
+  #   ]
+  #   bandit = ClassicBandit::Softmax.new(
+  #     arms: arms,
+  #     initial_temperature: 1.0,
+  #     k: 0.5
+  #   )
+  class Softmax
+    include ArmUpdatable
+
+    attr_reader :arms
+
+    def initialize(arms:, initial_temperature:, k:) # rubocop:disable Naming/MethodParameterName
+      @arms = arms
+      @initial_temperature = initial_temperature
+      @k = k
+
+      validate_parameters!
+    end
+
+    def select_arm
+      return @arms.sample if @arms.all? { |arm| arm.trials.zero? }
+
+      probabilities = @arms.map { |arm| softmax_score(arm, temperature) }
+      cumulative_prob = 0
+      random_value = rand
+
+      @arms.each_with_index do |arm, i|
+        cumulative_prob += probabilities[i]
+        return arm if random_value <= cumulative_prob
+      end
+
+      @arms.last
+    end
+
+    private
+
+    def softmax_score(arm, temperature)
+      exp_values = @arms.map { |a| Math.exp(a.mean_reward / temperature) }
+      Math.exp(arm.mean_reward / temperature) / exp_values.sum
+    end
+
+    def temperature
+      total_trials = @arms.sum(&:trials)
+      @initial_temperature / Math.log(@k * total_trials + 2)
+    end
+
+    def validate_parameters!
+      raise ArgumentError, "initial_temperature must be positive" unless @initial_temperature.positive?
+      raise ArgumentError, "k must be positive" unless @k.positive?
+    end
+  end
+end

data/lib/classic_bandit/thompson_sampling.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Implements Thompson Sampling algorithm for multi-armed bandit problems.
+  # Uses Beta-Bernoulli conjugate model, sampling from Beta distribution
+  # using Gamma random variables.
+  #
+  # @example Create and use Thompson Sampling
+  #   arms = [
+  #     ClassicBandit::Arm.new(id: 1, name: "banner_a"),
+  #     ClassicBandit::Arm.new(id: 2, name: "banner_b")
+  #   ]
+  #   bandit = ClassicBandit::ThompsonSampling.new(arms: arms)
+  class ThompsonSampling
+    include ArmUpdatable
+
+    attr_reader :arms
+
+    def initialize(arms:)
+      @arms = arms
+    end
+
+    def select_arm
+      return @arms.sample if @arms.all? { |arm| arm.trials.zero? }
+
+      @arms.max_by { |arm| ts_score(arm) }
+    end
+
+    private
+
+    def ts_score(arm)
+      return 0.0 if arm.trials.zero?
+      return 1.0 if arm.successes == arm.trials
+
+      x = gamma_random(arm.successes + 1)
+      y = gamma_random(arm.trials - arm.successes + 1)
+      x / (x + y)
+    end
+
+    def gamma_random(alpha) # rubocop:disable Metrics/AbcSize
+      return gamma_random(alpha + 1) * rand**(1.0 / alpha) if alpha < 1
+
+      # Marsaglia-Tsang method
+      d = alpha - 1.0 / 3
+      c = 1.0 / Math.sqrt(9 * d)
+
+      loop do
+        z = normal_random
+        v = (1 + c * z)**3
+        u = rand
+
+        return d * v if z > -1.0 / c && Math.log(u) < 0.5 * z * z + d * (1 - v + Math.log(v))
+      end
+    end
+
+    def normal_random
+      r = Math.sqrt(-2 * Math.log(rand))
+      theta = 2 * Math::PI * rand
+      r * Math.cos(theta)
+    end
+  end
+end

data/lib/classic_bandit/ucb1.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  # Implements the UCB1 (Upper Confidence Bound) algorithm for multi-armed bandit problems.
+  # This algorithm selects arms based on their mean rewards plus a confidence term,
+  # balancing exploration and exploitation without requiring an explicit epsilon parameter.
+  #
+  # @example Create and use UCB1 bandit
+  #   arms = [
+  #     ClassicBandit::Arm.new(id: 1, name: "banner_a"),
+  #     ClassicBandit::Arm.new(id: 2, name: "banner_b")
+  #   ]
+  #   bandit = ClassicBandit::Ucb1.new(arms: arms)
+  #   selected_arm = bandit.select_arm
+  #   bandit.update(selected_arm, reward: 1)
+  class Ucb1
+    include ArmUpdatable
+
+    # @return [Array<Arm>] Available arms for selection
+    attr_reader :arms
+
+    # Initialize a new UCB1 bandit
+    # @param arms [Array<Arm>] List of arms to choose from
+    def initialize(arms:)
+      @arms = arms
+    end
+
+    # Select an arm using the UCB1 algorithm.
+    # Initially tries each arm once, then uses UCB1 formula for selection.
+    # @return [Arm] Selected arm
+    def select_arm
+      # use untried arm if exists.
+      untried_arm = @arms.find { |arm| arm.trials.zero? }
+      return untried_arm if untried_arm
+
+      total_trials = @arms.sum(&:trials)
+      @arms.max_by { |arm| ucb_score(arm, total_trials) }
+    end
+
+    private
+
+    def ucb_score(arm, total_trials)
+      return Float::INFINITY if arm.trials.zero?
+
+      exploration_term = Math.sqrt(2 * Math.log(total_trials) / arm.trials)
+
+      arm.mean_reward + exploration_term
+    end
+  end
+end

data/lib/classic_bandit/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ClassicBandit
+  VERSION = "0.1.0"
+end

data/lib/classic_bandit.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+require_relative "classic_bandit/version"
+
+module ClassicBandit
+  class Error < StandardError; end
+  # Your code goes here...
+end
+
+loader = Zeitwerk::Loader.for_gem
+loader.setup
+loader.eager_load

data/sig/classic_bandit.rbs

@@ -0,0 +1,4 @@
+module ClassicBandit
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: classic_bandit
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kohei Tsuyuki
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2024-12-27 00:00:00.000000000 Z
+dependencies: []
+description: Implementation of classic multi-armed bandit algorithms in Ruby. Supports
+  Thompson Sampling, UCB1, and Epsilon-Greedy strategies with a simple, consistent
+  API for A/B testing and optimization tasks.
+email:
+- kotsuyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".devcontainer/devcontainer.json"
+- ".rspec"
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/classic_bandit.rb
+- lib/classic_bandit/arm.rb
+- lib/classic_bandit/arm_updatable.rb
+- lib/classic_bandit/epsilon_greedy.rb
+- lib/classic_bandit/softmax.rb
+- lib/classic_bandit/thompson_sampling.rb
+- lib/classic_bandit/ucb1.rb
+- lib/classic_bandit/version.rb
+- sig/classic_bandit.rbs
+homepage: https://github.com/t-chov/classic_bandit
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/t-chov/classic_bandit
+  source_code_uri: https://github.com/t-chov/classic_bandit
+  changelog_uri: https://github.com/t-chov/classic_bandit/blob/main/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.33
+signing_key: 
+specification_version: 4
+summary: A Ruby library for classic (non-contextual) multi-armed bandit algorithms
+test_files: []