mlanett-hive 0.3.0

data/lib/hive/configuration.rb

@@ -0,0 +1,251 @@
+# -*- encoding: utf-8 -*-
+
+require "optparse"
+
+=begin
+
+Evaluate a ruby configuration file in the context of a Hive Configuration instance.
+Offers a DSL to build the jobs as well as setting before/after-fork hooks.
+
+Hive configuration:
+
+env()
+set_env(ENV)
+--env=ENV
+Sets the environment.
+Used in pid file and log file naming.
+Defaults to RAILS_ENV || RACK_ENV || "test".
+
+chdir(DIR)
+--chdir=DIR
+Changes the working directory. Creates it if necessary.
+Takes effect immediately.
+Can only be set once. Has no effect if specified more than once.
+Defaults to /tmp/$NAME
+
+name()
+name=(NAME)
+--name=NAME
+Sets the name of the process.
+Defaults to the base name of the configuration file.
+Used in pid file and log file naming.
+
+--path=PATH
+add_path(PATH)
+Adds a path to the Ruby load path.
+Can be used multiple times.
+
+--require=LIB
+Requires a library or Ruby gem.
+Can be used multiple times.
+
+=end
+
+class Hive::Configuration
+
+  def self.parse( argv = ARGV )
+    us = new
+
+    optparse = OptionParser.new do |opts|
+      opts.banner = "Usage: #{__FILE__} [options]* configuration_file_rb"
+      opts.on( "-c", "--chdir DIR",   "Change working directory." )   { |d| us.chdir(d) }
+      opts.on( "-e", "--env ENV",     "Set environment (env).")       { |e| us.set_env(e) }
+      opts.on( "-h", "--help",        "Display this usage summary." ) { puts opts; exit }
+      opts.on( "-n", "--name NAME",   "Set daemon's name.")           { |n| us.set_name(n) }
+      opts.on( "-p", "--path PATH",   "Add to load path.")            { |d| us.add_path(d) }
+      opts.on( "-r", "--require LIB", "Require a library.")           { |l| us.require_lib(l) }
+      opts.on( "-s", "--script DSL",  "Include DSL script.")          { |s| us.load_script(s) }
+      opts.on( "-v", "--verbose",     "Print stuff out.")             { |s| us.verbose += 1 }
+      opts.on( "--dry-run",           "Don't launch the daemon.")     { us.dry_run = true }
+    end.parse!(argv)
+
+    while argv.any? && File.exists?(argv.first) do
+      us.load_file( argv.shift )
+    end
+
+    us.args = argv
+    us.finalize
+  end
+
+  include Hive::Log
+
+  attr :env
+  attr :root
+  attr :name, true
+  attr :verbose, true
+  attr :dry_run, true
+  attr :args, true
+  attr :before_forks
+  attr :after_forks
+
+  attr :defaults
+  attr :pools
+
+  # I'm not sure why this is so complicated.
+  class PoolEnumerator
+    def initialize pools
+      @pools = pools
+    end
+    include Enumerable
+    def each(&block)
+      them = @pools.each
+      it = nil
+      loop do
+        begin
+          it = them.next
+        rescue StopIteration => x
+          break
+        end
+        policy = Hive::Policy.resolve(it.last)
+        yield([ it.first, policy ])
+      end
+    end
+  end
+
+  def policies
+    PoolEnumerator.new(pools)
+  end
+
+  def initialize( filename = nil )
+    @verbose  = 0
+    @dry_run  = false
+    @defaults = {}
+    @pools    = {}
+    load_file(filename) if filename
+  end
+
+  def load_script(string)
+    log "Loading #{string}" if verbose >= 2
+    instance_eval(string)
+  end
+
+  def load_file(filename)
+    log "Loading #{filename}" if verbose >= 1
+    instance_eval(File.read(filename),filename)
+    if ! name then
+      n = File.basename(filename).sub(/\.[^.]*$/,'')
+      @name = n if n.size > 0 
+    end
+  end
+
+  def finalize()
+    if ! env then
+      @env = ( ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "test" )
+      log "Defaulting env to #{env}" if verbose >= 1
+    end
+    if ! name then
+      @name = "hive"
+      log "Defaulting name to #{name}" if verbose >= 1
+    end
+    if ! @root then
+      chdir(default_root)
+    end
+    log inspect if verbose >= 2
+    freeze
+    self
+  end
+
+  def options_for_daemon_spawn
+    mkdirp root, "#{root}/log", "#{root}/tmp", "#{root}/tmp/pids" if ! dry_run
+    return {
+      working_dir: root,
+      log_file:    "#{root}/log/#{name}_#{env}.log",
+      pid_file:    "#{root}/tmp/pids/#{name}_#{env}.pid",
+      sync_log:    local?
+    }
+  end
+
+  def args_for_daemon_spawn
+    args + [self]
+  end
+
+  # ----------------------------------------------------------------------------
+  # DSL
+  # ----------------------------------------------------------------------------
+
+  def set_env(env)
+    @env = env
+  end
+
+  def set_name(name)
+    @name = name
+  end
+
+  # takes effect immediately
+  def chdir(path)
+    if ! @root then
+      p = File.expand_path(path)
+      mkdirp(p) if ! dry_run
+      Dir.chdir(p)
+      log "Changed working directory (root) to #{p}" if verbose >= 1
+      @root = p
+    else
+      log "Warning: working directory already set to #{root}; not changing to #{path}"
+    end
+  end
+
+  # takes effect immediately
+  def add_path(path)
+    p = File.expand_path(path)
+    log "Added #{p} to load path" if verbose >= 2
+    $:.push(p) unless $:.member?(p)
+  end
+
+  # convenience for -r on the command line
+  def require_lib(r)
+    require(r)
+    log "Required #{r}" if verbose >= 2
+  end
+
+  def set_default(key,value)
+    # values which are arrays get merged, but nil will overwrite
+    case value
+    when Array
+      @defaults[key] = (@defaults[key] || []) + value
+    else
+      @defaults[key] = value
+    end
+  end
+
+  def set_defaults(options)
+    options.each { |k,v| set_default(k,v) }
+  end
+  
+  def add_pool( name, options = {} )
+    before_forks = (options[:before_forks] || []) + (self.before_forks || [])
+    after_forks  = (options[:after_forks] || []) + (self.after_forks || [])
+    options      = defaults.merge(options).merge before_forks: before_forks, after_forks: after_forks
+    pools[name]  = options
+    log "Added pool for #{name}" if verbose == 1
+    log "Added pool for #{name} with #{options}" if verbose >= 2
+  end
+
+  def before_fork(&block)
+    @before_forks ||= []
+    @before_forks << block
+  end
+
+  def after_fork(&block)
+    @after_forks ||= []
+    @after_forks << block
+  end
+
+  # ----------------------------------------------------------------------------
+  private
+  # ----------------------------------------------------------------------------
+
+  LOCAL_ENVS = [ "development", "test" ]
+
+  def local?
+    LOCAL_ENVS.member?(env)
+  end
+
+  def default_root
+    local? ? "." : "/tmp/#{name}"
+  end
+
+  def mkdirp(*ps)
+    ps.each { |p| Dir.mkdir(p) if ! Dir.exists?(p) }
+  end
+
+end

data/lib/hive/idler.rb

@@ -0,0 +1,81 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+
+Idler wraps some other callable (a proc or object which responds to #call)
+The callable should return a falsy value when it did nothing,
+or a truthy value when it did something.
+The idler will sleep when there is nothing to do.
+
+=end
+
+class Hive::Idler
+  
+  MIN_SLEEP = 0.125
+  MAX_SLEEP = 1.0
+  
+  attr :sleep
+  
+  def initialize( callable = nil, options = {}, &callable_block )
+    @callable = callable || callable_block
+    raise unless @callable.respond_to?(:call)
+
+    @max_sleep = options[:max_sleep] || MAX_SLEEP
+    raise if @max_sleep <= 0
+
+    @min_sleep = options[:min_sleep] || MIN_SLEEP
+    raise if @min_sleep <= 0
+    raise if @max_sleep < @min_sleep
+
+    @sleep     = nil
+  end
+  
+  def call( *args, &block )
+    
+    result = call_with_wakefulness( @callable, *args, &block )
+
+    if result then
+      wake
+    else
+      sleep_more
+    end
+
+    return result
+  end
+
+  def call_with_wakefulness( callable, *args, &block )
+    begin
+      callable.call(*args,&block)
+    rescue Exception                          # when errors occur,
+      @sleep = @min_sleep                     # reduce sleeping almost all the way (but not to 0)
+      raise                                   # do not consume any exceptions
+    end
+  end
+
+  def sleep_more
+    if @sleep then
+      @sleep = [ @sleep * 2, @max_sleep ].min
+    else
+      @sleep = @min_sleep
+    end
+    Kernel.sleep(@sleep) if @sleep          # Interrupt will propogate through sleep().
+  end
+
+  def wake
+    @sleep = nil
+  end
+
+  module Utilities
+    # execute test repeatedly, until timeout, or until test returns true
+    def wait_until( timeout = 1, &test )
+      tester = Hive::Idler.new(test)
+      finish = Time.now.to_f + timeout
+      loop do
+        break if tester.call
+        break if finish < Time.now.to_f
+      end
+    end
+  end
+  extend Utilities
+
+end # Hive::Idler

data/lib/hive/key.rb

@@ -0,0 +1,48 @@
+# -*- encoding: utf-8 -*-
+
+=begin
+
+  A key uniquely identifies a worker.
+
+=end
+
+class Hive::Key
+
+  attr :name
+  attr :pid
+  attr :host
+
+  def initialize( name, pid, host = Hive::Key.local_host )
+    @name = name
+    @pid  = pid.to_i
+    @host = host
+  end
+
+  def ==(other)
+    self.equal?(other) ||
+    ( name == other.name && pid == other.pid && host == other.host )
+  end
+
+  # e.g. processor-1234@foo.example.com
+  def to_s
+    "%s-%i@%s" % [ name, pid, host ]
+  end
+
+  def self.parse(key_string)
+    key_string =~ /^(.*)-([0-9]+)@([^@]+)$/ or raise MalformedKey.new(key_string)
+    new( $1, $2, $3 )
+  end
+
+  # ----------------------------------------------------------------------------
+  # Utilities
+  # ----------------------------------------------------------------------------
+
+  # @returns something like foo.example.com
+  def self.local_host
+    @local_host ||= `hostname`.chomp.strip
+  end
+
+  class MalformedKey < StandardError
+  end
+
+end

data/lib/hive/lifecycle_observer.rb

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+
+class Hive::LifecycleObserver < Hive::Utilities::ObserverBase
+
+  attr :key
+  attr :registry
+
+  def initialize( key, registry )
+    @key      = key
+    @registry = registry
+  end
+
+  def worker_started
+    registry.register( key )
+  end
+
+  def worker_heartbeat( upcount = 0 )
+    registry.update( key )
+  end
+
+  def worker_stopped
+    registry.unregister( key )
+  end
+
+end # Hive::LifecycleObserver

data/lib/hive/log.rb

@@ -0,0 +1,29 @@
+module Hive::Log
+  
+  def log( *args )
+    logger.print(format_for_logging(*args))
+    logger.flush
+  end
+
+  def format_for_logging( *args )
+    message = [
+      #(Time.now.strftime "%Y%m%d%H%M%S"),
+      Time.now.to_i,
+      " [",
+      Process.pid,
+      (Thread.current[:name] || Thread.current.object_id unless Thread.current == Thread.main),
+      "] ",
+      args.join(", "),
+      "\n"
+    ].compact.join
+  end
+
+  def logger
+    @logger ||= STDOUT
+  end
+
+  def logger=( other )
+    @logger = other
+  end
+
+end # Hive::Log

data/lib/hive/messager.rb

@@ -0,0 +1,217 @@
+# -*- encoding: utf-8 -*-
+
+require "digest/md5"
+require "json"
+
+=begin
+
+  Messager is used to send messages between processes, and receive responses.
+  Messager messages are asynchronous and not ordered.
+
+=end
+
+class Hive::Messager
+
+  attr :callbacks
+  attr :storage
+  attr :my_address
+  attr :to_address
+
+  # @param options[:to_address] is optional
+  # @param options[:my_address] is required
+  def initialize( storage, options = {} )
+    @callbacks  = {}
+    @storage    = storage
+    @to_address = options[:to_address]
+    @my_address = options[:my_address] or raise "must specify my address"
+    # type checking
+    storage.get("test")
+  end
+
+  # write to another queue
+  # @param options[:to] is required if :to_address was not given
+  # @returns an id
+  def send( body, options = {} )
+    to        = options[:to] || to_address or raise "must specify to address"
+    from      = options[:from] || my_address or raise "must specify from address"
+    now       = options[:at] || Time.now
+    message   = Message.new( options.merge( to: to, from: my_address, at: now, body: body ) )
+    blob      = message.to_json
+
+    storage.queue_add( queue_name(to), blob, now.to_i )
+    message.id
+  end
+
+  # register a handler for a given id
+  # the handler is removed when it is called
+  def expect( match, &callback )
+    @callbacks[match] = callback
+    self
+  end
+
+  # sends a new message to the original message source and with reply_to_id from the original message
+  # @param options[:to] must be the original message
+  # @e.g. reply "Ok", to: question
+  def reply( body, options )
+    original = options[:to] or raise "must reply to: message"
+    send( body, to: original.from, reply_to_id: original.id )
+  end
+
+  # @param reply_block takes (body, headers)
+  def expect_reply( src_id, &reply_block )
+    raise
+  end
+
+  # read from my queue
+  # check to see if there are any messages, and dispatch them
+  # @returns true if processed a message, false otherwise
+  def receive()
+    now  = Time.now.to_i
+    json = storage.queue_pop( queue_name, now )
+    if json then
+      message  = Message.parse(json)
+      callback = find_callback( message )
+      callback.call( message )
+      true
+    else
+      false
+    end
+  end
+
+  # ----------------------------------------------------------------------------
+  # Message contains the body and critical headers for Messager
+  # ----------------------------------------------------------------------------
+
+  class Message
+
+    attr :to           # destination host
+    attr :from         # source host
+    attr :at           # timestamp of message generation
+    attr :body         # JSON-compatible
+    attr :id           # autogenerated if not supplied
+    attr :reply_to_id  # optional
+
+    def initialize( data )
+      data         = ::Hive::Messager.symbolize(data)
+      @to          = data[:to] or raise "must specify to address"
+      @from        = data[:from] or raise "must specify from address"
+      @at          = (data[:at] || Time.now).to_f
+      @body        = data[:body]
+      @id          = data[:id] || Digest::MD5.hexdigest([from,at,body].join)
+      @reply_to_id = data[:reply_to_id]
+    end
+
+    def to_hash
+      blob = { to: to, from: from, at: at, body: body, id: id }
+      blob[:reply_to_id] = reply_to_id if reply_to_id
+      blob
+    end
+
+    def to_json
+      to_hash.to_json
+    end
+
+    def to_s
+      to_json
+    end
+
+    def self.parse( json )
+        new( JSON.parse(json) )
+    end
+
+  end # Message
+
+  # ----------------------------------------------------------------------------
+  # Utilities
+  # ----------------------------------------------------------------------------
+
+  def self.stringify(map)
+    Hash[ map.map { |k,v| [ k.to_s, v ] } ]
+  end
+
+  def self.symbolize(map)
+    Hash[ map.map { |k,v| [ k.to_sym, v ] } ]
+  end
+
+  # ----------------------------------------------------------------------------
+  protected
+  # ----------------------------------------------------------------------------
+
+  def queue_name( other_address = nil )
+    if other_address then
+      "messages:#{other_address}"
+    else
+      @queue_name ||= "messages:#{my_address}"
+    end
+  end
+
+  # ----------------------------------------------------------------------------
+  # Match
+  # ----------------------------------------------------------------------------
+
+  class NoMatch < Exception
+  end
+
+  class Counter
+    def match
+      @value ||= 0
+      @value += 1
+    end
+    def fail
+      @value = nil
+      raise NoMatch
+    end
+    def value
+      raise NoMatch if !@value
+      @value
+    end
+  end
+
+  def find_callback( message )
+    best_result = nil
+    best_score = nil
+    callbacks.each do |match,callback|
+      begin
+        counter = Counter.new
+        compare_match( message, match, counter )
+        if !best_score || counter.value > best_score then
+          best_score  = counter.value
+          best_result = callback
+        end
+      rescue NoMatch
+        # next
+      end
+    end
+    return best_result if best_result
+    raise NoMatch
+  end
+
+  def compare_match( message, match, counter )
+    case match
+    when String, Regexp
+      compare( message.body, match, counter )
+    when Hash
+      compare( message.to_hash, match, counter )
+    end
+  end
+
+  def compare( item, match, counter )
+    case match
+    when Numeric
+      return item.kind_of?(Numeric) && item == match ? counter.match : counter.fail
+    when String
+      return item.kind_of?(String) && item == match ? counter.match : counter.fail
+    when Regexp
+      return item.kind_of?(String) && item =~ match ? counter.match : counter.fail
+    when Hash
+      counter.fail if ! item.kind_of?(Hash)
+      match.each do |k,v|
+        counter.fail if ! item.has_key?(k)
+        compare( item[k], match[k], counter )
+      end
+    else
+      raise "Can not compare using #{match.inspect}"
+    end
+  end
+
+end