furnish 0.0.4 → 0.1.0

data/lib/furnish/scheduler.rb

@@ -32,6 +32,16 @@ module Furnish
     #
     attr_accessor :force_deprovision
 
+    ##
+    #
+    # When true, calling #run or #recover also installs a SIGINFO (Ctrl+T in the
+    # terminal on macs) and SIGUSR2 handler which can be used to get
+    # information on the status of what's solved and what's working.
+    #
+    # Default is true.
+    #
+    attr_accessor :signal_handler
+
     #
     # Instantiate the Scheduler.
     #
@@ -43,6 +53,8 @@ module Furnish
       @working_threads    = { }
       @queue              = Queue.new
       @vm                 = Furnish::VM.new
+      @recovering         = false
+      @signal_handler     = true
     end
 
     #
@@ -63,6 +75,28 @@ module Furnish
       end
     end
 
+    #
+    # Is recovery running? See #recover.
+    #
+    def recovering?
+      @recovering
+    end
+
+    #
+    # Is recovery necessary? See #recover.
+    #
+    def needs_recovery?
+      needs_recovery.count > 0
+    end
+
+    #
+    # A map of group name to Furnish::ProvisionerGroup for groups that failed
+    # their #startup or #shutdown. See #recover for more information
+    #
+    def needs_recovery
+      vm.need_recovery
+    end
+
     #
     # Schedule a group of VMs for provision. This takes a group name, which is a
     # string, an array of provisioner objects, and a list of string dependencies.
@@ -76,6 +110,9 @@ module Furnish
       schedule_provisioner_group(group)
     end
 
+    alias s schedule_provision
+    alias sched schedule_provision
+
     #
     # Schedule a provision with a Furnish::ProvisionerGroup. Works exactly like
     # Furnish::Scheduler#schedule_provision otherwise.
@@ -94,8 +131,12 @@ module Furnish
       vm.sync_waiters do |waiters|
         waiters.add(group.name)
       end
+
+      return true
     end
 
+    alias << schedule_provisioner_group
+
     #
     # Sleep until this list of dependencies are resolved. In parallel mode, will
     # raise if an exception occurred while waiting for these resources. In
@@ -121,24 +162,11 @@ module Furnish
     # and #running? and #stop to control and monitor the threads this class
     # manages.
     #
-    # This call also installs a SIGINFO (Ctrl+T in the terminal on macs) and
-    # SIGUSR2 handler which can be used to get information on the status of
-    # what's solved and what's working. You can disable this functionality by
-    # passing `false` as the first argument.
-    #
-    def run(install_handler=true)
+    def run
       # short circuit if we're not serial and already running
-      return if @solver_thread and !@serial
-
-      if install_handler
-        handler = lambda do |*args|
-          Furnish.logger.puts ["solved:", vm.solved.to_a].inspect
-          Furnish.logger.puts ["working:", vm.working.to_a].inspect
-          Furnish.logger.puts ["waiting:", vm.waiters.to_a].inspect
-        end
+      return if running?
 
-        %w[USR2 INFO].each { |sig| trap(sig, &handler) if Signal.list[sig] }
-      end
+      install_handler if signal_handler
 
       if @serial
         service_resolved_waiters
@@ -151,6 +179,69 @@ module Furnish
       end
     end
 
+    #
+    # Initiate recovery. While running, #recovering? will be true.
+    #
+    # Recovery will step through all the items in #needs_recovery and attempt
+    # to recover them according to Furnish::ProvisionerGroup#recover. If
+    # recovery succeeds, the items will be in the solved formula and
+    # effectively provisioned. They will also be removed from the
+    # needs_recovery information.
+    #
+    # If recovery fails, #needs_recovery will not be touched (but the state at
+    # which recovery starts the next attempt may be different for those
+    # groups). Additionally, the return value of this method will be keyed by
+    # the group name, and an exception or false depending on what we got back
+    # during recovery. It is strongly recommended you check #needs_recovery? or
+    # the return value after calling this to locate flapping groups.
+    #
+    # Recovery is a serial process and blocks the main thread. It also installs
+    # a signal handler if #signal_handler is set. It does not interrupt or stop
+    # the scheduler, but note that in serial mode, the scheduler will likely
+    # already be stopped by the time you are able to call recovery. In
+    # threaded mode, this means any dependencies that are able to be
+    # provisioned after a successful recovery of a group will automatically
+    # start provisioning.
+    #
+    def recover
+      install_handler if signal_handler
+
+      @recovering = true
+
+      failures = { }
+
+      needs_recovery.keys.each do |k|
+        begin
+          group = vm.groups[k]
+          result = group.recover(force_deprovision)
+          vm.groups[k] = group
+
+          if result
+            needs_recovery.delete(k)
+            @queue << k
+          else
+            failures[k] = false
+          end
+        rescue => e
+          failures[k] = e
+        end
+      end
+
+      if @serial
+        begin
+          queue_loop
+        rescue => e
+          if_debug do
+            puts "During recovery, serial mode, encountered: #{e}: #{e.message}"
+          end
+        end
+      end
+
+      @recovering = false
+
+      return failures
+    end
+
     #
     # Instructs the scheduler to stop. Note that this is not an interrupt, and
     # the queue will still be exhausted before terminating.
@@ -260,9 +351,11 @@ module Furnish
     #
     def with_timeout(do_loop=true)
       Timeout.timeout(1) do
-        dead_working = @working_threads.values.reject(&:alive?)
-        if dead_working.size > 0
-          dead_working.map(&:join)
+        dead_working = @working_threads.reject { |k,v| v.alive? }
+        if dead_working.keys.size > 0
+          dead_working.each do |k, t|
+            @working_threads.delete(k)
+          end
         end
 
         yield
@@ -319,7 +412,7 @@ module Furnish
     #
     def resolve_waiters
       vm.sync_waiters do |waiters|
-        waiters.replace(waiters.to_set - (@working_threads.keys.to_set + vm.solved.to_set))
+        waiters.replace(waiters.to_set - (vm.working.to_set + vm.solved.to_set))
       end
     end
 
@@ -352,12 +445,12 @@ module Furnish
     # Similar to #startup -- just a shim to talk to a specific ProvisionerGroup
     #
     def shutdown(group_name)
-      provisioner = vm.groups[group_name]
+      group = vm.groups[group_name]
 
       # if we can't find the provisioner, we probably got asked to clean up
       # something we never scheduled. Just ignore that.
-      if provisioner and can_deprovision?(group_name)
-        provisioner.shutdown(@force_deprovision)
+      if group and can_deprovision?(group_name)
+        group.shutdown({}, force_deprovision)
       end
     end
 
@@ -381,9 +474,20 @@ module Furnish
               # HACK: just give the working check something that will always work.
               #       Probably should just mock it.
               @working_threads[group_name] = Thread.new { sleep }
-              startup(group_name)
+              begin
+                startup(group_name)
+              rescue => e
+                vm.need_recovery[group_name] = e
+                raise e
+              end
             else
-              @working_threads[group_name] = Thread.new { startup(group_name) }
+              @working_threads[group_name] = Thread.new do
+                begin
+                  startup(group_name)
+                rescue => e
+                  vm.need_recovery[group_name] = e
+                end
+              end
             end
           end
         end
@@ -402,6 +506,7 @@ module Furnish
     # any threads managing it.
     #
     def delete_group(group_name)
+      vm.need_recovery.delete(group_name)
       vm.solved.delete(group_name)
       vm.sync_waiters do |waiters|
         waiters.delete(group_name)
@@ -412,5 +517,23 @@ module Furnish
       vm.dependencies.delete(group_name)
       vm.groups.delete(group_name)
     end
+
+    #
+    # Installs our signal handler -- see #signal_handler.
+    #
+    def install_handler
+      handler = lambda do |*args|
+        # XXX See Palsy#with_t and Palsy#no_lock for why this is necessary.
+        Palsy.instance.no_lock do
+          Furnish.logger.puts ["solved:", vm.solved.to_a].inspect
+          Furnish.logger.puts ["working:", vm.working.to_a].inspect
+          Furnish.logger.puts ["waiting:", vm.waiters.to_a].inspect
+          Furnish.logger.puts ["provisioning:", vm.working.to_a.map { |w| [w, vm.groups[w].group_state['action'], vm.groups[w].group_state['provisioner']] }]
+          Furnish.logger.puts ["needs recovery:", needs_recovery.keys]
+        end
+      end
+
+      %w[USR2 INFO].each { |sig| trap(sig, &handler) if Signal.list[sig] }
+    end
   end
 end

data/lib/furnish/version.rb

@@ -1,4 +1,4 @@
 module Furnish
   # The current version of Furnish.
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/furnish/vm.rb

@@ -14,6 +14,8 @@ module Furnish
     attr_reader :working
     # the set of groups waiting to be provisioned.
     attr_reader :waiters
+    # the set of groups that need recovery, and the exceptions they threw (if any)
+    attr_reader :need_recovery
 
     #
     # Create a new VM object. Should only happen in the Scheduler.
@@ -21,6 +23,7 @@ module Furnish
     def initialize
       @groups        = Palsy::Map.new('vm_groups', 'provisioner_group')
       @dependencies  = Palsy::Map.new('vm_groups', 'dependency_group')
+      @need_recovery = Palsy::Map.new('vm_groups', 'need_recovery')
       @solved        = Palsy::Set.new('vm_scheduler', 'provisioned')
       @working       = Palsy::Set.new('vm_scheduler', 'working')
       @waiters       = Palsy::Set.new('vm_scheduler', 'waiters')

data/test/dummy_classes.rb

@@ -0,0 +1,275 @@
+#
+# Several dummy class mutations (inherited from Furnish::Provisioner::Dummy) we
+# use in tests.
+#
+
+require 'furnish/provisioners/dummy'
+
+Dummy = Furnish::Provisioner::Dummy unless defined? Dummy
+
+#
+# FIXME move all these dummy classes to their own file
+#
+# FIXME Probably should generate these classes
+#
+
+class ReturnsInfoDummy < Dummy
+  def startup(args={})
+    retval = { :startup_blah => [1] }
+    super(retval)
+    return retval
+  end
+
+  def shutdown(args={})
+    retval = { :shutdown_blah => [1] }
+    super(retval)
+    return retval
+  end
+end
+
+class AcceptsIntegerBarDummy < Dummy
+  configure_startup do
+    accepts :bar, "bar", Integer
+  end
+end
+
+class YieldsIntegerBarDummy < Dummy
+  configure_startup do
+    yields :bar, "bar", Integer
+  end
+end
+
+class AcceptsStringBarDummy < Dummy
+  configure_startup do
+    accepts :bar, "bar", String
+  end
+end
+
+class YieldsStringBarDummy < Dummy
+  configure_startup do
+    yields :bar, "bar", String
+  end
+end
+
+class AcceptsFooDummy < Dummy
+  configure_startup do
+    accepts :foo, "foo", String
+  end
+end
+
+class YieldsFooDummy < Dummy
+  configure_startup do
+    yields :foo, "foo", String
+  end
+end
+
+class RequiresBarDummy < Dummy
+  configure_startup do
+    requires :bar, "bar", Integer
+  end
+end
+
+class YieldsFooBarDummy < Dummy
+  configure_startup do
+    yields :bar, "bar", Integer
+    yields :foo, "foo", String
+  end
+end
+
+class RequiresBarAcceptsFooDummy < Dummy
+  configure_startup do
+    requires :bar, "bar", Integer
+    accepts :foo, "foo", String
+  end
+end
+
+class ShutdownAcceptsIntegerBarDummy < Dummy
+  configure_shutdown do
+    accepts :bar, "bar", Integer
+  end
+end
+
+class ShutdownYieldsIntegerBarDummy < Dummy
+  configure_shutdown do
+    yields :bar, "bar", Integer
+  end
+end
+
+class ShutdownAcceptsStringBarDummy < Dummy
+  configure_shutdown do
+    accepts :bar, "bar", String
+  end
+end
+
+class ShutdownYieldsStringBarDummy < Dummy
+  configure_shutdown do
+    yields :bar, "bar", String
+  end
+end
+
+class ShutdownAcceptsFooDummy < Dummy
+  configure_shutdown do
+    accepts :foo, "foo", String
+  end
+end
+
+class ShutdownYieldsFooDummy < Dummy
+  configure_shutdown do
+    yields :foo, "foo", String
+  end
+end
+
+class ShutdownRequiresBarDummy < Dummy
+  configure_shutdown do
+    requires :bar, "bar", Integer
+  end
+end
+
+class ShutdownYieldsFooBarDummy < Dummy
+  configure_shutdown do
+    yields :bar, "bar", Integer
+    yields :foo, "foo", String
+  end
+end
+
+class ShutdownRequiresBarAcceptsFooDummy < Dummy
+  configure_shutdown do
+    requires :bar, "bar", Integer
+    accepts :foo, "foo", String
+  end
+end
+
+class StartFailDummy < Dummy
+  def startup(args={ })
+    super
+    false
+  end
+end
+
+class StopFailDummy < Dummy
+  def shutdown(args={ })
+    super
+    false
+  end
+end
+
+class StartExceptionDummy < Dummy
+  def startup(args={ })
+    super
+    raise "ermagherd startup"
+  end
+end
+
+class StopExceptionDummy < Dummy
+  def shutdown(args={ })
+    super
+    raise "ermagherd shutdown"
+  end
+end
+
+class APIDummy < Furnish::Provisioner::API
+  furnish_property :foo, "does things with foo", Integer
+  furnish_property "a_string"
+  attr_accessor :bar
+end
+
+class BadDummy < Furnish::Provisioner::Dummy
+  attr_accessor :name
+
+  # this retardation lets us make it look like furnish_group_name doesn't exist
+  def respond_to?(meth, include_all=true)
+    super unless [:furnish_group_name, :furnish_group_name=].include?(meth)
+  end
+end
+
+class SleepyDummy < Dummy
+  def startup(args={ })
+    sleep 1
+    super
+  end
+end
+
+class SleepyFailingDummy < SleepyDummy
+  def startup(args={ })
+    super
+    return false
+  end
+end
+
+class BrokenRecoverAPIDummy < Dummy
+  allows_recovery
+end
+
+class RecoverableDummy < Dummy
+  allows_recovery
+
+  def startup(args={ })
+    super
+    run_state[__method__] = @recovered
+    return @recovered
+  end
+
+  def shutdown(args={ })
+    super
+    run_state[__method__] = @recovered
+    return @recovered
+  end
+
+  def recover(state, args)
+    @recovered = { state => true }
+    return true
+  end
+end
+
+class RaisingRecoverableDummy < Dummy
+  allows_recovery
+
+  def startup(args={ })
+    super
+    run_state[__method__] = @recovered
+    raise unless @recovered
+    return @recovered
+  end
+
+  def shutdown(args={ })
+    super
+    run_state[__method__] = @recovered
+    raise unless @recovered
+    return @recovered
+  end
+
+  def recover(state, args)
+    @recovered = { state => true }
+    return true
+  end
+end
+
+class FailedRecoverDummy < Dummy
+  allows_recovery
+
+  def startup(args={ })
+    super
+    return run_state[__method__] = false
+  end
+
+  def shutdown(args={ })
+    super
+    return run_state[__method__] = false
+  end
+
+  def recover(state, args)
+    return run_state[__method__] = false
+  end
+end
+
+class ReturnsDataDummy < Dummy
+  def startup(args={ })
+    super
+    return({ :started => 1 })
+  end
+
+  def shutdown(args={ })
+    super
+    return({ :stopped => 1 })
+  end
+end