collective 0.2.0

data/lib/collective/mocks/storage.rb

@@ -0,0 +1,108 @@
+# -*- encoding: utf-8 -*-
+
+class Collective::Mocks::Storage
+  
+  def initialize
+    @storage = {}
+  end
+
+  def reconnect_after_fork
+    # nop
+  end
+
+  # Simple values
+
+  def put( key, value )
+    @storage[key] = value
+  end
+  
+  def get( key )
+    @storage[key]
+  end
+  
+  def del( key )
+    @storage.delete( key )
+  end
+
+  # Sets
+
+  def set_add( key, value )
+    @storage[key] ||= []
+    @storage[key] << value unless @storage[key].member?(value)
+  end
+  
+  def set_size( key )
+    (@storage[key] || [] ).size
+  end
+  
+  def set_remove( key, value )
+    (@storage[key] || [] ).delete( value )
+  end
+
+  def set_member?( key, value )
+    (@storage[key] || []).member?( value )
+  end
+
+  def set_get_all( key )
+    @storage[key] || []
+  end
+
+  # Maps
+
+  def map_set( key, name, value )
+    @storage[key] ||= {}
+    @storage[key][name] = value
+  end
+  
+  def map_get( key, name )
+    (@storage[key] || {}) [name]
+  end
+  
+  def map_get_all_keys( key )
+    (@storage[key] || {}).keys
+  end
+  
+  def map_size( key )
+    (@storage[key] || {} ).size
+  end
+  
+  def map_del( key )
+    @storage.delete( key )
+  end
+
+  # Priority Queue
+
+  def queue_add( queue_name, item, score )
+    queue = @storage[queue_name] ||= []
+    queue << [ item, score ]
+    queue.sort_by! { |it| it.last }
+  end
+
+  # pop the lowest item from the queue IFF it scores <= max_score
+  def queue_pop( queue_name, max_score = Time.now.to_i )
+    queue = @storage[queue_name] || []
+    return nil if queue.size == 0
+    if queue.first.last <= max_score then
+      queue.shift.first
+    else
+      nil
+    end
+  end
+
+  def queue_pop_sync( queue_name, max_score = Time.now.to_i, options = {} )
+    timeout  = options[:timeout] || 1
+    deadline = Time.now.to_f + timeout
+
+    loop do
+      result = queue_pop( queue_name, max_score )
+      return result if result
+
+      raise Timeout::Error if Time.now.to_f > deadline
+    end
+  end
+
+  def queue_del( queue_name )
+    @storage.delete( queue_name )
+  end
+  
+end # Collective::Mocks::Storage

data/lib/collective/monitor.rb

@@ -0,0 +1,58 @@
+# -*- encoding: utf-8 -*-
+require 'ruby-debug'
+
+class Collective::Monitor
+
+  include Collective::Log
+
+  attr :pools
+
+  def initialize( configuration )
+    @pools = configuration.jobs.map do |kind,options|
+      pool = Collective::Pool.new( kind, Collective::Policy.resolve(options) )
+    end
+  end
+
+
+  def monitor
+    status = {}
+
+    job = ->() do
+      changed = false
+      pools.each do |pool|
+
+        log pool.name
+        previous = status[pool.name]
+        current  = pool.synchronize log: true
+
+        if previous != current then
+          status[pool.name] = current
+          changed = true
+        end
+
+      end
+      changed
+    end
+
+    job = Collective::Idler.new( job, min_sleep: 1, max_sleep: 10 )
+
+    ok = true
+    trap("TERM") { ok = false }
+    while ok do
+      job.call
+    end
+  end # monitor
+
+  def stop_all
+    pools.each do |pool|
+      pool.stop_all
+    end
+  end
+
+  def restart
+    pools.each do |pool|
+      pool.restart
+    end
+  end
+
+end

data/lib/collective/policy.rb

@@ -0,0 +1,60 @@
+# -*- encoding: utf-8 -*-
+
+require "ostruct"
+
+class Collective::Policy
+
+  DEFAULTS = {
+    pool_min_workers:       1,
+    pool_max_workers:       10,
+    worker_idle_max_sleep:  64.0,
+    worker_idle_min_sleep:  0.125,
+    worker_idle_spin_down:  900,
+    worker_none_spin_up:    86400,
+    worker_max_jobs:        100,    # a worker should automatically exit after this many jobs
+    worker_max_lifetime:    1000,   # a worker should automatically exit after this time
+    worker_late:            10,     # a worker is overdue after this time with no heartbeat
+    worker_hung:            100,    # a worker will be killed after this time
+    storage:                :mock,
+    observers:              []
+  }
+
+  class Instance
+
+    # including options[:policy] will merge over these options
+    def initialize( options = {} )
+      if options[:policy] then
+        policy   = options.delete(:policy)
+        defaults = policy.dup
+      else
+        defaults = DEFAULTS
+      end
+
+      options  = Hash[ options.map { |k,v| [ k.to_sym, v ] } ] # poor man's symbolize keys
+      @options = defaults.merge( options )
+    end
+
+    def storage
+      Collective::Utilities::StorageBase.resolve @options[:storage]
+    end
+
+    def method_missing( symbol, *arguments )
+      @options[symbol.to_sym]
+    end
+
+    def dup
+      @options.dup
+    end
+
+  end # Instance
+
+  class << self
+
+    def resolve( options = {} )
+      # this will dup either an Instance or a Hash
+      Collective::Policy::Instance.new(options.dup)
+    end
+
+  end # class
+
+end # Collective::Policy

data/lib/collective/pool.rb

@@ -0,0 +1,180 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+
+  A pool is a collection of workers, each of which is a separate process.
+  All workers are of the same kind (class).
+
+=end
+
+class Collective::Pool
+
+  include Collective::Log
+
+  attr :kind      # job class
+  attr :name
+  attr :policy
+  attr :registry
+  attr :storage   # where to store worker details
+
+  def initialize( kind, policy_prototype = {} )
+    if kind.kind_of?(Array) then
+      kind, policy_prototype = kind.first, kind.last
+    end
+    @kind     = kind
+    @policy   = Collective::Policy.resolve(policy_prototype) or raise
+    @name     = @policy.name || kind.name or raise Collective::ConfigurationError, "Pool or Job must have a name"
+    @storage  = policy.storage
+    @registry = Collective::Registry.new( name, storage )
+
+    # type checks
+    policy.pool_min_workers
+    registry.workers
+  end
+
+
+  # @param options[:log] can be true
+  # @returns the checked worker lists
+  def synchronize( options = {} )
+    do_log = options.delete(:log)
+    raise if options.size > 0
+
+    checklist  = registry.checked_workers( policy )
+    live_count = checklist.live.size
+
+    if do_log then
+      check_live_workers( checklist )
+      check_late_workers( checklist )
+      check_hung_workers( checklist )
+      check_dead_workers( checklist )
+    end
+
+    if (need = policy.pool_min_workers - live_count) > 0 then
+      # launch workers
+      need.times do
+        spawn wait: true
+      end
+
+    elsif (excess = live_count - policy.pool_max_workers) > 0 then
+      # spin down some workers
+      # try to find LOCAL workers to spin down first
+      locals = checklist.live.select { |k| k.host == Collective::Key.local_host }
+      if locals.size > 0 then
+        reap locals.first, wait: true
+      else
+        reap checklist.live.first, wait: true
+      end
+    end
+
+    checklist = registry.checked_workers( policy )
+  end
+
+
+  def mq
+    @mq ||= begin
+      key = Collective::Key.new( "#{name}-pool", Process.pid )
+      me  = Collective::Messager.new storage, my_address: key
+    end
+  end
+
+
+  # tell all workers to quit
+  def stop_all
+    checklist = registry.checked_workers( policy )
+    checklist.live.each { |key| reap(key) }
+    checklist.late.each { |key| reap(key) }
+    checklist.hung.each { |key| reap(key) }
+    checklist.dead.each { |key| reap(key) }
+  end
+
+
+  # this really should be protected but it's convenient to be able to force a spawn
+  # param options[:wait] can true to wait until after the process is spawned
+  def spawn( options = {} )
+    wait = options.delete(:wait)
+    raise if options.size > 0
+
+    if ! wait then
+      Collective::Worker.spawn kind, registry: registry, policy: policy, name: name
+      return
+    end
+
+    before = registry.checked_workers( policy ).live
+
+    Collective::Worker.spawn kind, registry: registry, policy: policy, name: name
+
+    Collective::Idler.wait_until( 10 ) do
+      after = registry.checked_workers( policy ).live
+      diff  = ( after - before ).select { |k| k.host == Collective::Key.local_host }
+      diff.size > 0
+    end
+  end
+
+
+  # shut down a worker
+  def reap( key, options = {} )
+    wait = options.delete(:wait)
+    raise if options.size > 0
+
+    if key.host == Collective::Key.local_host then
+      ::Process.kill( "TERM", key.pid )
+      Collective::Utilities::Process.wait_and_terminate key.pid, timeout: 10
+    else
+      mq.send "Quit", to: key
+    end
+
+    if wait then
+      Collective::Idler.wait_until( 10 ) do
+        live = registry.checked_workers( policy ).live
+        ! live.member? key
+      end
+    end
+  end
+
+  # ----------------------------------------------------------------------------
+  protected
+  # ----------------------------------------------------------------------------
+
+  def check_live_workers( checked )
+    if live = checked.live and live.size > 0 then
+      log "Live worker count #{live.size}; members: #{live.inspect}"
+      live.size
+    else
+      0
+    end
+  end
+
+
+  def check_late_workers( checked )
+    if late = checked.late and late.size > 0 then
+      log "Late worker count #{late.size}; members: #{late.inspect}"
+      late.size
+    else
+      0
+    end
+  end
+
+
+  def check_hung_workers( checked )
+    if hung = checked.hung and hung.size > 0 then
+      log "Hung worker count #{hung.size}"
+      hung.each do |key|
+        log "Killing #{key}"
+        Collective::Utilities::Process.wait_and_terminate( key.pid )
+        registry.unregister(key)
+      end
+    end
+    0
+  end
+
+
+  def check_dead_workers( checked )
+    if dead = checked.dead and dead.size > 0 then
+      log "Dead worker count #{dead.size}; members: #{dead.inspect}"
+      dead.size
+    else
+      0
+    end
+  end
+
+end # Collective::Pool

data/lib/collective/redis/storage.rb

@@ -0,0 +1,142 @@
+# -*- encoding: utf-8 -*-
+
+require "redis"
+require "redis-namespace"
+require "timeout"
+
+class Collective::Redis::Storage
+
+  def initialize( redis = nil )
+    self.redis = redis if redis
+  end
+
+  def reconnect_after_fork
+    redis.client.disconnect
+  end
+
+
+  # Simple values
+
+  def put( key, value )
+    redis.set( key, value )
+  end
+  
+  def get( key )
+    redis.get( key )
+  end
+
+  def del( key )
+    redis.del( key )
+  end
+
+  # Sets
+
+  def set_add( set_name, value )
+    redis.sadd( set_name, value )
+  end
+
+  def set_size( set_name )
+    redis.scard( set_name )
+  end
+
+  def set_remove( set_name, value )
+    redis.srem( set_name, value )
+  end
+
+  def set_get_all( set_name )
+    redis.smembers( set_name )
+  end
+
+  def set_member?( set_name, value )
+    redis.sismember( set_name, value )
+  end
+
+  # Priority Queue
+
+  def queue_add( queue_name, item, score = Time.now.to_i )
+    score = score.to_f
+    begin
+      redis.zadd( queue_name, score, item )
+    rescue Exception => x
+      raise x, "Failed zadd( #{queue_name.inspect}, #{score.inspect}, #{item.inspect} ) because of an error: #{x.message}", x.backtrace
+    end
+  end
+
+  # pop the lowest item from the queue IFF it scores <= max_score
+  def queue_pop( queue_name, max_score = Time.now.to_i )
+    # Option 1: zrange, check score, accept or discard
+    # Option 2: zrangebyscore with limit, then zremrangebyrank
+
+    redis.watch( queue_name )
+    it = redis.zrangebyscore( queue_name, 0, max_score, limit: [0,1] ).first
+    if it then
+      ok = redis.multi { |r| r.zremrangebyrank( queue_name, 0, 0 ) }
+      it = nil if ! ok
+    else
+      redis.unwatch
+    end
+    it
+  end
+
+  def queue_pop_sync( queue_name, max_score = Time.now.to_i, options = {} )
+    timeout  = options[:timeout] || 1
+    deadline = Time.now.to_f + timeout
+
+    loop do
+      result = queue_pop( queue_name, max_score )
+      return result if result
+
+      raise Timeout::Error if Time.now.to_f > deadline
+    end
+  end
+
+  def queue_del( queue_name )
+    redis.del( queue_name )
+  end
+
+  # Maps
+
+  def map_set( map_name, key, value )
+    redis.hset( map_name, key, value )
+  end
+
+  def map_get( map_name, key )
+    redis.hget( map_name, key )
+  end
+
+  def map_get_all_keys( map_name )
+    redis.hkeys( map_name )
+  end
+
+  def map_size( map_name )
+    redis.hlen( map_name )
+  end
+
+  def map_del( map_name, key )
+    redis.hdel( map_name, key )
+  end
+
+  # ----------------------------------------------------------------------------
+  # Redis
+  # ----------------------------------------------------------------------------
+
+  # @param redis_client can only be set once
+  def redis=( redis_or_options )
+    raise Collective::ConfigurationError if @redis
+
+    case redis_or_options
+    when Hash
+      options   = redis_or_options.dup
+      namespace = options.delete(:namespace)
+      @redis    = Redis.connect(options)
+      @redis    = Redis::Namespace.new( namespace, redis: @redis ) if namespace
+    else
+      @redis    = redis_or_options
+    end
+  end
+
+  def redis
+    @redis ||= ::Redis.connect( url: "redis://127.0.0.1:6379/1" )
+  end
+
+end # Collective::Redis::Storage