finite_mdp 0.0.1

data/lib/finite_mdp/solver.rb

@@ -0,0 +1,344 @@
+require 'narray'
+
+#
+# Find optimal values and policies using policy iteration and/or value
+# iteration. The methods here are suitable for finding deterministic policies
+# for infinite-horizon problems.
+#
+# The computations are carried out on an intermediate form of the given model,
+# which is stored using nested arrays:
+#   model[state_num][action_num] = [[next_state_num, probability, reward], ...]
+# The solver assigns numbers to each state and each action automatically. Note
+# that the successor state data are stored in sparse format, and any transitions
+# that are in the given model but have zero probability are not stored.
+#
+# TODO implement backward induction for finite horizon problems
+#
+# TODO maybe implement a 'dense' storage format for models with many successor
+# states, probably as a different solver class
+#
+class FiniteMDP::Solver
+  #
+  # @param [Model] model
+  #
+  # @param [Float] discount in (0, 1]
+  #
+  # @param [Hash<state, action>, nil] policy initial policy; if nil, an
+  #        arbitrary action is selected for each state
+  #
+  # @param [Hash<state, Float>] value initial value for each state; defaults to
+  #        zero for every state
+  #
+  def initialize model, discount, policy=nil, value=Hash.new(0)
+    @model = model
+    @discount = discount
+
+    # get the model data into a more compact form for calculation; this means
+    # that we number the states and actions for faster lookups (avoid most of
+    # the hashing); the 'next states' map is still stored in sparse format
+    # (that is, as a hash)
+    model_states = model.states
+    state_to_num = Hash[model_states.zip((0...model_states.size).to_a)]
+    @array_model = model_states.map {|state|
+      model.actions(state).map {|action|
+        model.next_states(state, action).map {|next_state|
+          pr = model.transition_probability(state, action, next_state)
+          [state_to_num[next_state], pr, 
+            model.reward(state, action, next_state)] if pr > 0
+        }.compact
+      }
+    }
+
+    # convert initial values and policies to compact form
+    @array_value = model_states.map {|state| value[state]}
+    if policy
+      action_to_num = model_states.map{|state|
+        actions = model.actions(state)
+        Hash[actions.zip((0...actions.size).to_a)]
+      }
+      @array_policy = action_to_num.zip(model_states).
+                            map {|a_to_n, state| a_to_n[policy[state]]}
+    else
+      # default to the first action, arbitrarily
+      @array_policy = [0]*model_states.size
+    end
+
+    raise 'some initial values are missing' if
+      @array_value.any? {|v| v.nil?}
+    raise 'some initial policy actions are missing' if
+      @array_policy.any? {|a| a.nil?}
+
+    @policy_A = nil
+  end
+
+  #
+  # @return [Model] the model being solved; read only; do not change the model
+  # while it is being solved
+  #
+  attr_reader :model
+
+  # 
+  # Current value estimate for each state.
+  #
+  # The result is converted from the solver's internal representation, so you
+  # cannot affect the solver by changing the result. 
+  #
+  # @return [Hash<state, Float>] from states to values; read only; any changes
+  # made to the returned object will not affect the solver
+  #
+  def value
+    Hash[model.states.zip(@array_value)]
+  end
+
+  #
+  # Current estimate of the optimal action for each state.
+  #
+  # @return [Hash<state, action>] from states to actions; read only; any changes
+  # made to the returned object will not affect the solver
+  #
+  def policy
+    Hash[model.states.zip(@array_policy).map{|state, action_n|
+      [state, model.actions(state)[action_n]]}]
+  end
+
+  #
+  # Refine the estimate of the value function for the current policy. This is
+  # done by iterating the Bellman equations; see also {#evaluate_policy_exact}
+  # for a different approach.
+  #
+  # This is the 'policy evaluation' step in Figure 4.3 of Sutton and Barto
+  # (1998).
+  #
+  # @return [Float] largest absolute change (over all states) in the value
+  # function
+  #
+  def evaluate_policy
+    delta = 0.0
+    @array_model.each_with_index do |actions, state_n|
+      next_state_ns = actions[@array_policy[state_n]]
+      new_value = backup(next_state_ns)
+      delta = [delta, (@array_value[state_n] - new_value).abs].max
+      @array_value[state_n] = new_value
+    end
+    delta
+  end
+
+  #
+  # Evaluate the value function for the current policy by solving a linear
+  # system of n equations in n unknowns, where n is the number of states in the
+  # model.
+  #
+  # This routine currently uses dense linear algebra, so it requires that the
+  # full n-by-n matrix be stored in memory. This may be a problem for moderately
+  # large n.
+  #
+  # All of the coefficients (A and b in Ax = b) are computed first call, but
+  # subsequent calls recompute only those rows for which the policy has changed
+  # since the last call.
+  #
+  # @return [nil]
+  #
+  def evaluate_policy_exact
+    if @policy_A
+      # update only those rows for which the policy has changed
+      @policy_A_action.zip(@array_policy).
+        each_with_index do |(old_action_n, new_action_n), state_n|
+        next if old_action_n == new_action_n
+        update_policy_Ab state_n, new_action_n
+      end
+    else
+      # initialise the A and the b for Ax = b
+      num_states = @array_model.size
+      @policy_A = NMatrix.float(num_states, num_states)
+      @policy_A_action = [-1]*num_states
+      @policy_b = NVector.float(num_states)
+
+      @array_policy.each_with_index do |action_n, state_n|
+        update_policy_Ab state_n, action_n
+      end
+    end
+
+    value = @policy_b / @policy_A # solve linear system
+    @array_value = value.to_a
+    nil
+  end
+
+  #
+  # Make policy greedy with respect to the current value function.
+  #
+  # This is the 'policy improvement' step in Figure 4.3 of Sutton and Barto
+  # (1998).
+  # 
+  # @return [Boolean] false iff the policy changed for any state
+  #
+  def improve_policy
+    stable = true
+    @array_model.each_with_index do |actions, state_n|
+      a_max = nil
+      v_max = -Float::MAX
+      actions.each_with_index do |next_state_ns, action_n|
+        v = backup(next_state_ns)
+        if v > v_max
+          a_max = action_n
+          v_max = v
+        end
+      end
+      raise "no feasible actions in state #{state_n}" unless a_max
+      stable = false if @array_policy[state_n] != a_max
+      @array_policy[state_n] = a_max
+    end
+    stable
+  end
+
+  #
+  # A single iteration of value iteration.
+  #
+  # This is the algorithm from Figure 4.5 of Sutton and Barto (1998). It is
+  # mostly equivalent to calling {#evaluate_policy} and then {#improve_policy},
+  # but it is somewhat more efficient.
+  #
+  # @return [Float] largest absolute change (over all states) in the value
+  # function
+  #
+  def value_iteration_single
+    delta = 0.0
+    @array_model.each_with_index do |actions, state_n|
+      a_max = nil
+      v_max = -Float::MAX
+      actions.each_with_index do |next_state_ns, action_n|
+        v = backup(next_state_ns)
+        if v > v_max
+          a_max = action_n
+          v_max = v
+        end
+      end
+      delta = [delta, (@array_value[state_n] - v_max).abs].max
+      @array_value[state_n] = v_max
+      @array_policy[state_n] = a_max
+    end
+    delta
+  end
+
+  #
+  # Value iteration; call {#value_iteration_single} up to
+  # <tt>max_iters</tt> times until the largest change in the value function
+  # (<tt>delta</tt>) is less than <tt>tolerance</tt>.
+  #
+  # @param [Float] tolerance small positive number
+  #
+  # @param [Integer, nil] max_iters terminate after this many iterations, even 
+  #        if the value function has not converged; nil means that there is
+  #        no limit on the number of iterations
+  #
+  # @return [Boolean] true iff iteration converged to within tolerance
+  #
+  def value_iteration tolerance, max_iters=nil
+    delta = Float::MAX
+    num_iters = 0
+    loop do
+      delta = value_iteration_single
+      num_iters += 1
+
+      break if delta < tolerance
+      break if max_iters && num_iters > max_iters
+    end
+    delta < tolerance
+  end
+
+  #
+  # Solve with policy iteration using approximate (iterative) policy evaluation.
+  #
+  # @param [Float] value_tolerance small positive number; the policy evaluation
+  #        phase ends if the largest change in the value function
+  #        (<tt>delta</tt>) is below this tolerance
+  #
+  # @param [Integer, nil] max_value_iters terminate the policy evaluation
+  #        phase after this many iterations, even if the value function has not
+  #        converged; nil means that there is no limit on the number of
+  #        iterations in each policy evaluation phase
+  #
+  # @param [Integer, nil] max_policy_iters terminate after this many
+  #        iterations, even if a stable policy has not been obtained; nil means
+  #        that there is no limit on the number of iterations
+  #
+  # @return [Boolean] true iff a stable policy was obtained
+  #
+  def policy_iteration value_tolerance, max_value_iters=nil,
+    max_policy_iters=nil
+
+    stable = false
+    num_policy_iters = 0
+    loop do
+      # policy evaluation
+      num_value_iters = 0
+      loop do
+        value_delta = evaluate_policy
+        num_value_iters += 1
+
+        break if value_delta < value_tolerance
+        break if max_value_iters && num_value_iters > max_value_iters
+      end
+
+      # policy improvement
+      stable = improve_policy
+      num_policy_iters += 1
+      break if stable
+      break if max_policy_iters && num_policy_iters > max_policy_iters
+    end
+    stable
+  end
+
+  #
+  # Solve with policy iteration using exact policy evaluation.
+  #
+  # @param [Integer, nil] max_iters terminate after this many
+  #        iterations, even if a stable policy has not been obtained; nil means
+  #        that there is no limit on the number of iterations
+  #
+  # @return [Boolean] true iff a stable policy was obtained
+  #
+  def policy_iteration_exact max_iters=nil
+    stable = false
+    num_iters = 0
+    loop do
+      evaluate_policy_exact
+      stable = improve_policy
+      num_iters += 1
+      break if stable
+      break if max_iters && num_iters > max_iters
+    end
+    stable
+  end
+
+  private
+
+  #
+  # Updated value estimate for a state with the given successor states.
+  #
+  def backup next_state_ns
+    next_state_ns.map {|next_state_n, probability, reward|
+      probability*(reward + @discount*@array_value[next_state_n])
+    }.inject(:+)
+  end
+
+  #
+  # Update the row in A the entry in b (in Ax=b) for the given state; see
+  # {#evaluate_policy_exact}.
+  #
+  def update_policy_Ab state_n, action_n
+    # clear out the old values for state_n's row
+    @policy_A[true, state_n] = 0.0
+
+    # set new values according to state_n's successors under the current policy
+    b_n = 0
+    next_state_ns = @array_model[state_n][action_n]
+    next_state_ns.each do |next_state_n, probability, reward|
+      @policy_A[next_state_n, state_n] = -@discount*probability
+      b_n += probability*reward
+    end
+    @policy_A[state_n, state_n] += 1
+    @policy_A_action[state_n] = action_n
+    @policy_b[state_n] = b_n
+  end
+end
+

data/lib/finite_mdp/table_model.rb

@@ -0,0 +1,122 @@
+#
+# A finite markov decision process model for which the states, actions,
+# transition probabilities and rewards are specified as a table. This is a
+# common way of specifying small models.
+#
+# The states and actions can be arbitrary objects; see notes for {Model}.
+#
+class FiniteMDP::TableModel
+  include FiniteMDP::Model
+
+  #
+  # @param [Array<[state, action, state, Float, Float]>] rows each row is
+  #  [state, action, next state, probability, reward]
+  #
+  def initialize rows
+    @rows = rows
+  end
+
+  #
+  # @return [Array<[state, action, state, Float, Float]>] each row is [state,
+  #  action, next state, probability, reward]
+  #
+  attr_accessor :rows
+
+  #
+  # States in this model; see {Model#states}.
+  #
+  # @return [Array<state>] not empty; no duplicate states
+  #
+  def states
+    @rows.map{|row| row[0]}.uniq
+  end
+
+  #
+  # Actions that are valid for the given state; see {Model#actions}.
+  #
+  # @param [state] state
+  #
+  # @return [Array<action>] not empty; no duplicate actions
+  #
+  def actions state
+    @rows.map{|row| row[1] if row[0] == state}.compact.uniq
+  end
+
+  #
+  # Possible successor states after taking the given action in the given state;
+  # see {Model#next_states}.
+  # 
+  # @param [state] state
+  #
+  # @param [action] action
+  #
+  # @return [Array<state>] not empty; no duplicate states
+  #
+  def next_states state, action
+    @rows.map{|row| row[2] if row[0] == state && row[1] == action}.compact
+  end 
+
+  #
+  # Probability of the given transition; see {Model#transition_probability}.
+  #
+  # @param [state] state
+  #
+  # @param [action] action
+  #
+  # @param [state] next_state
+  #
+  # @return [Float] in [0, 1]; zero if the transition is not in the table
+  # 
+  def transition_probability state, action, next_state
+    @rows.map{|row| row[3] if row[0] == state &&
+      row[1] == action && row[2] == next_state}.compact.first || 0
+  end
+
+  #
+  # Reward for a given transition; see {Model#reward}.
+  #
+  # @param [state] state
+  #
+  # @param [action] action
+  #
+  # @param [state] next_state
+  #
+  # @return [Float, nil] nil if the transition is not in the table
+  #
+  def reward state, action, next_state
+    @rows.map{|row| row[4] if row[0] == state &&
+      row[1] == action && row[2] == next_state}.compact.first
+  end
+
+  #
+  # @return [String] can be quite large
+  #
+  def inspect
+    rows.map(&:inspect).join("\n")
+  end
+
+  #
+  # Convert any model into a table model.
+  #
+  # @param [Model] model
+  #
+  # @param [Boolean] sparse do not store rows for transitions with zero
+  #        probability
+  #
+  # @return [TableModel]
+  #
+  def self.from_model model, sparse=true
+    rows = []
+    model.states.each do |state|
+      model.actions(state).each do |action|
+        model.next_states(state, action).each do |next_state|
+          pr = model.transition_probability(state, action, next_state)
+          rows << [state, action, next_state,  pr,
+            model.reward(state, action, next_state)] if pr > 0 || !sparse
+        end
+      end
+    end
+    FiniteMDP::TableModel.new(rows)
+  end
+end
+

data/lib/finite_mdp/vector_valued.rb

@@ -0,0 +1,46 @@
+#
+# Define an object's hash code and equality (in the sense of <tt>eql?</tt>)
+# according to its array representation (<tt>to_a</tt>). See notes for {Model}
+# for why this might be useful.
+#
+# A class that includes this module must define <tt>to_a</tt>.
+#
+# @example
+#
+#   class MyPoint 
+#     include FiniteMDP::VectorValued
+#
+#     def initialize x, y
+#       @x, @y = x, y
+#     end
+#
+#     attr_accessor :x, :y
+#
+#     # must implement to_a to make VectorValued work
+#     def to_a
+#       [x, y]
+#     end
+#   end
+#
+#   MyPoint.new(0, 0).eql?(MyPoint.new(0, 0)) #=> true as expected
+#
+module FiniteMDP::VectorValued
+  #
+  # Redefine hashing based on +to_a+.
+  #
+  # @return [Integer]
+  #
+  def hash
+    self.to_a.hash
+  end
+
+  #
+  # Redefine equality based on +to_a+.
+  #
+  # @return [Boolean]
+  #
+  def eql? state
+    self.to_a.eql? state.to_a
+  end
+end
+

data/lib/finite_mdp/version.rb

@@ -0,0 +1,3 @@
+module FiniteMDP
+  VERSION = '0.0.1'
+end

data/lib/finite_mdp.rb

@@ -0,0 +1,14 @@
+require 'enumerator'
+
+require 'finite_mdp/version'
+require 'finite_mdp/vector_valued'
+require 'finite_mdp/model'
+require 'finite_mdp/hash_model'
+require 'finite_mdp/table_model'
+require 'finite_mdp/solver'
+
+# TODO maybe for efficiency it would be worth including a special case for
+# models in which rewards depend only on the state -- a few minor
+# simplifications are possible in the solver, but it won't make a huge
+# difference.
+