collective 0.2.0

data/lib/collective/registry.rb

@@ -0,0 +1,123 @@
+# -*- encoding: utf-8 -*-
+
+require 'ostruct'
+
+=begin
+
+  A registry knows how to lookup up and register workers.
+
+=end
+
+class Collective::Registry
+
+  attr :name
+  attr :storage
+
+  def initialize( name, storage )
+    @name    = name    or raise
+    @storage = storage or raise
+
+    # type checking
+    name.encoding
+  end
+
+
+  def reconnect_after_fork
+    @storage.reconnect_after_fork
+  end
+
+
+  def register( key )
+    key = key.to_s
+    storage.set_add( workers_key, key )
+    storage.put( status_key(key), Time.now.to_i )
+  end
+
+
+  def update( key )
+    key = key.to_s
+    storage.set_add( workers_key, key ) if ! storage.set_member?( workers_key, key )
+    storage.put( status_key(key), Time.now.to_i )
+  end
+
+
+  def unregister( key )
+    key = key.to_s
+    storage.del( status_key(key) )
+    storage.set_remove( workers_key, key )
+  end
+
+  # ----------------------------------------------------------------------------
+  # Query API
+  # ----------------------------------------------------------------------------
+
+  # @returns an array of key strings
+  # NOTICE this will include keys for workers on all hosts
+  def workers
+    all = storage.set_get_all( workers_key )
+    raise "Not a Set: #{workers_key} (#{all.class})" unless all.kind_of?(Array)
+    all.map { |key_string| Collective::Key.parse(key_string) }
+  end
+
+
+  def checked_workers( policy )
+    groups = { live: [], late: [], hung: [], dead: [] }
+    check_workers(policy) do |key, status|
+      groups[status] << key
+    end
+    OpenStruct.new(groups)
+  end
+
+
+  # This method can be slow so it takes a block for incremental processing.
+  # @param block takes entry, status in [ :live, :late, :hung, :dead ]
+  # @param options[:all] = true to get keys across all hosts
+  def check_workers( policy, options = nil, &block )
+    workers.each do |key|
+      heartbeat = storage.get( status_key(key.to_s) ).to_i
+      status    = heartbeat_status( policy, heartbeat )
+      yield( key, status )
+    end
+  end
+
+  # ----------------------------------------------------------------------------
+  protected
+  # ----------------------------------------------------------------------------
+
+  def heartbeat_status( policy, heartbeat )
+    if heartbeat > 0 then
+      age = now - heartbeat.to_i
+      if age >= policy.worker_hung then
+        :hung
+      elsif age >= policy.worker_late
+        :late
+      else
+        :live
+      end
+    else
+      :dead
+    end
+  end
+
+
+  # easier to test if we can stub Time.now
+  def now
+    Time.now.to_i
+  end
+
+
+  def policy
+    @policy ||= Collective::Policy.resolve
+  end
+
+
+  def workers_key
+    @workers_key ||= "collective:#{name}:workers"
+  end
+
+
+  def status_key( key )
+    "collective:#{name}:worker:#{key}"
+  end
+
+end # Collective::Registry

data/lib/collective/squiggly.rb

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+
+module Squiggly
+
+  ADJECTIVES = ["able", "afraid", "bad", "bitter", "brave", "bright", "busy", "careful", "cheap", "clean", "clear", "clever", "close", "cloudy", "cold", "comfortable", "cool", "cute", "dangerous", "dapper", "dark", "dead", "deep", "difficult", "dirty", "dry", "early", "empty", "exciting", "expensive", "fair", "famous", "far", "fast", "fat", "fine", "flat", "free", "free", "free", "fresh", "full", "funny", "good", "great", "happy", "hard", "healthy", "heavy", "high", "hungry", "important", "interesting", "kind", "large", "late", "lazy", "light", "long", "loud", "low", "lucky", "narrow", "near", "noisy", "old", "polite", "proud", "quick", "quiet", "rich", "sad", "safe", "salty", "scared", "short", "slimey", "slow", "small", "soft", "sour", "strong", "strong", "sweet", "thick", "thirsty", "tidy", "useful", "warm", "weak", "weak", "whole", "windy"]
+  ANIMALS    = ["aardvarks", "ants", "badgers", "bats", "bears", "bees", "butterflies", "canaries", "cattle", "chickens", "chihuahuas", "clams", "cockles", "crabs", "crows", "deer", "dogs", "donkeys", "doves", "dragonflies", "ducks", "ferrets", "flies", "foxes", "frogs", "geese", "gerbils", "goats", "guinea pigs", "hamsters", "hares", "hawks", "hedgehogs", "herons", "horses", "hummingbirds", "kingfishers", "lobsters", "mice", "moles", "moths", "mussles", "newts", "otters", "owls", "oysters", "parrots", "peafowl", "pheasants", "pigeons", "pigs", "pikes", "platypuses", "rabbits", "rats", "robins", "rooks", "salmons", "sheep", "snails", "snakes", "sparrows", "spiders", "squid", "squirrels", "starlings", "stoats", "swans", "toads", "trouts", "wasps", "weasels"]
+  VERBS      = ["cried", "cuddled", "danced", "drove", "engaged", "felt", "floundered", "fought", "hid", "hopped", "hugged", "jumped", "kissed", "knitted", "listened", "loitered", "married", "played", "played", "ran", "sang", "sat", "snuggled", "talked", "walked", "went", "whimpered", "whimpered", "whispered"]
+  ADVERBS    = ["cordially", "easily", "jovially", "merrily", "quickly"]
+
+  def sentence
+    count = rand(8) + 2
+    [ count, ADJECTIVES.sample, ANIMALS.sample, VERBS.sample, ADVERBS.sample ].join(" ")
+  end
+
+  def subject
+    [ ADJECTIVES.sample, ANIMALS.sample ].join(" ")
+  end
+
+  extend Squiggly
+end # Squiggly

data/lib/collective/utilities/airbrake_observer.rb

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+  
+  Wrapps a callable and issues feedback.
+  - Error
+  
+=end
+
+require "airbrake"
+
+class Collective::Utilities::AirbrakeObserver < Collective::Utilities::ObserverBase
+
+  attr :it # job
+  attr :me # worker
+  
+  def initialize( observed, callable = nil, &callable_block )
+    @it = callable || callable_block
+    @me = observed
+  end
+
+  def job_error(x)
+    Airbrake.notify(x)
+  end
+
+end # Collective::Utilities::AirbrakeObserver

data/lib/collective/utilities/hoptoad_observer.rb

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+  
+  Wrapps a callable and issues feedback.
+  - Error
+  
+=end
+
+require "hoptoad_notifier"
+
+class Collective::Utilities::HoptoadObserver < Collective::Utilities::ObserverBase
+
+  attr :it # job
+  attr :me # worker
+  
+  def initialize( observed, callable = nil, &callable_block )
+    @it = callable || callable_block
+    @me = observed
+  end
+
+  def job_error(x)
+    HoptoadNotifier.notify(x)
+  end
+
+end # Collective::Utilities::HoptoadObserver

data/lib/collective/utilities/log_observer.rb

@@ -0,0 +1,40 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+  
+  Wrapps a callable and issues feedback.
+  - Started
+  - Hearbeat
+  - Error
+  - Stopped
+  
+=end
+
+class Collective::Utilities::LogObserver < Collective::Utilities::ObserverBase
+  
+  include Collective::Log
+
+  def initialize( filename = nil )
+    if filename then
+      self.logger = File.open(filename,"a") # append or create, write only
+    end
+  end
+
+  def worker_started( *args )
+    log "#{subject} has started"
+  end
+  
+  def worker_heartbeat( *args )
+    log "#{subject} is still alive"
+  end
+  
+  def job_error(x)
+    log "Warning: #{subject} experienced a job failure due to an error:#{x.inspect}"
+    log x.backtrace
+  end
+  
+  def worker_stopped( *args )
+    log "#{subject} has stopped"
+  end
+
+end # Collective::Utilities::LogObserver

data/lib/collective/utilities/observeable.rb

@@ -0,0 +1,18 @@
+# -*- encoding: utf-8 -*-
+
+module Collective::Utilities::Observeable
+
+  def add_observer(o)
+    o.focus(self)
+    (@observers ||= []).push(o)
+  end
+
+  def notify( *details )
+    if @observers then
+      @observers.each do |observer|
+        observer.notify( self, *details )
+      end
+    end
+  end
+
+end

data/lib/collective/utilities/observer_base.rb

@@ -0,0 +1,59 @@
+# -*- encoding: utf-8 -*-
+
+class Collective::Utilities::ObserverBase
+
+  attr :subject
+
+  # can implement notify( subject, *details )
+  # or can use this implementation
+
+  # It is possible but unlikely that I would want an observer to observe multiple subjects.
+  def focus( subject )
+    @subject = subject
+  end
+
+  def notify( subject, *details )
+    if self.respond_to?(details.first) then
+      details = details.dup
+      self.send( details.shift, *details )
+    end
+  end
+
+  # factory_or_observer can be something which responds to #notify
+  # or a block which responds to #call and can return a factory_or_observer
+  # or a class which can be instantiated
+  # or a string which can be resolved to a class
+  # or an array which can be resolved to a class with parameters
+  def self.resolve( factory_or_observer, *args )
+    case
+    when factory_or_observer.respond_to?(:notify)
+      factory_or_observer
+    when factory_or_observer.respond_to?(:call)
+      resolve(factory_or_observer.call(*args))
+    else
+      case factory_or_observer
+      when :airbrake
+        resolve(Collective::Utilities::AirbrakeObserver,*args)
+      when :hoptoad
+        resolve(Collective::Utilities::HoptoadObserver,*args)
+      when :log
+        resolve(Collective::Utilities::LogObserver,*args)
+      when Class
+        factory_or_observer.new(*args)
+      when String
+        resolve(Collective.resolve_class(factory_or_observer.to_s),*args)
+      when Array
+        args = factory_or_observer.dup
+        fobs = args.shift
+        factory = resolve( fobs, *args )
+      else
+        return factory_or_observer # assume it supports the notifications natively
+      end
+    end
+  end
+
+  def self.camelize(s)
+    s.to_s.gsub(/(?:^|_|\s)(.)/) { $1.upcase }
+  end
+
+end # Collective::Utilities::ObserverBase

data/lib/collective/utilities/process.rb

@@ -0,0 +1,82 @@
+# -*- encoding: utf-8 -*-
+
+module Collective::Utilities::Process
+  
+  def wait_until_deadline( pid, deadline )
+    status   = nil
+    interval = 0.125
+    begin # execute at least once to get status
+      dummy, status = wait2_now(pid)
+      break if status
+      #log "Waiting for #{pid}", "Sleeping #{interval}" if false
+      sleep(interval)
+      interval *= 2 if interval < 1.0
+    end while Time.now.to_f < deadline
+    status
+  end
+
+
+  def wait_and_terminate( pid, options = {} )
+    #log "Monitoring job #{pid}"
+    timeout = options[:timeout] || 1024
+    signal  = options[:signal] || "TERM"
+    
+    status = wait_until_deadline( pid, Time.now.to_f + timeout )
+    return status if status
+    
+    #log "Job #{pid} is overdue, killing"
+    
+    ::Process.kill( signal, pid )
+    status = wait_until_deadline( pid, Time.now.to_f + 1 )
+    return status if status
+    
+    ::Process.kill( "KILL", pid ) if ! status
+    dummy, status = wait2_now(pid)
+    status
+  end
+
+
+  def fork_and_detach( options = {}, &action)
+    # Fork twice. First child doesn't matter. The second is our favorite.
+    pid1 = fork do
+      ::Process.setsid
+      exit if fork
+      redirect_stdio( options[:stdout] )
+      action.call
+    end
+
+    # We must call waitpid on the first child to keep it from turning into a zombie
+    ::Process.waitpid(pid1)
+  end
+
+
+  def wait2_now( pid )
+    ::Process.wait2( pid, ::Process::WNOHANG )
+  rescue Errno::ECHILD # No child processes
+    return [ nil, 0 ]
+  end
+
+
+  def redirect_stdio( stdout )
+    STDIN.reopen "/dev/null"
+    if stdout then
+      mask = File.umask(0000)
+      file = File.new( stdout, "a" ) # append or create, write only
+      File.umask( mask )
+      STDOUT.reopen( file )
+    else
+      STDOUT.reopen "/dev/null"
+    end
+    STDERR.reopen(STDOUT)
+  end
+
+
+  def alive?( pid )
+    !! ::Process.kill( 0, pid )
+  rescue Errno::ESRCH
+    false
+  end
+
+  extend Collective::Utilities::Process
+
+end # Collective::Utilities::Process

data/lib/collective/utilities/signal_hook.rb

@@ -0,0 +1,47 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+  Allows you to add hooks without worrying about restoring the chain when you are done.
+  Hooks can be nested.
+  Only the most deeply nested handler will be called.
+
+  Usage:
+    hook = SignalHook.trap("TERM") { ... my term handler }
+    hook.attempt do
+      ... my long action
+    end
+    # at this point, term handler has been removed from chain
+    
+    or
+    SignalHook.trap("QUIT") { foo.quit! }.attempt { foo.run }
+=end
+
+class Collective::SignalHook
+
+  attr_writer :local, :chain
+
+  def trigger
+    if @local then
+      @local.call
+    elsif @chain
+      @chain.call
+    end
+  end
+
+  def attempt( &block )
+    yield
+  ensure
+    @local = false
+  end
+
+  class << self
+    def trap( signal, &block )
+      hook       = new
+      hook.local = block
+      previous   = Signal.trap( signal ) { hook.trigger }
+      hook.chain = previous if previous && previous.kind_of?(Proc)
+      hook
+    end # trap
+  end
+
+end # Collective::SignalHook