actory 0.0.2

data/example/prime_parallel_in_threads.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'parallel'
+require 'progressbar'
+
+METHOD = "prime"
+ARGS   = (1..1000).to_a
+
+require_relative "../lib/actory/receiver/plugin/example_#{METHOD}"
+require_relative './lib/benchmark'
+
+res = []
+processor_count = Parallel.processor_count
+pbar = ProgressBar.new(METHOD, processor_count)
+
+ret, time = Benchmark.measure do
+  begin
+    plugin = Actory::Receiver::Plugin.new
+    res << Parallel.map(ARGS, :in_threads => processor_count) do |arg|
+      begin
+        pbar.set pbar.current + 1 if pbar.current <= processor_count
+      rescue
+      end
+      plugin.send(METHOD, arg)
+    end
+    res.each do |v|
+      puts "returned #{v}"
+    end
+  rescue => e
+    @num == nil ? @num = 0 : @num += 1
+    puts e
+    puts $@
+    sleep 1
+    retry if @num < 2
+  end
+end
+
+puts " => #{ret}"
+puts " => time = #{time} sec"

data/example/prime_single.rb

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'parallel'
+require 'progressbar'
+
+METHOD = "prime"
+ARGS   = (1..1000).to_a
+
+require_relative "../lib/actory/receiver/plugin/example_#{METHOD}"
+require_relative './lib/benchmark'
+
+res = []
+processor_count = Parallel.processor_count
+pbar = ProgressBar.new(METHOD, ARGS.count)
+
+ret, time = Benchmark.measure do
+  begin
+    plugin = Actory::Receiver::Plugin.new
+    res << ARGS.map do |arg|
+      begin
+        pbar.set pbar.current + 1 if pbar.current <= ARGS.count
+      rescue
+      end
+      plugin.send(METHOD, arg)
+    end
+    res.each do |v|
+      puts "returned #{v}"
+    end
+  rescue => e
+    @num == nil ? @num = 0 : @num += 1
+    puts e
+    puts $@
+    sleep 1
+    retry if @num < 2
+  end
+end
+
+puts " => #{ret}"
+puts " => time = #{time} sec"

data/lib/actory.rb

@@ -0,0 +1,66 @@
+require 'yaml'
+
+module Actory
+
+  class Base
+    GLOBAL = YAML.load_file(File.expand_path("../../config/global.yml", __FILE__))
+
+    def self.get_logger_output(type="stdout", target=nil)
+      return nil unless type
+      out = nil
+      file = File.open(target, "a") if %w[file both].include?(type)
+      case type.downcase
+      when "stdout"
+        out = STDOUT
+      when "file"
+        out = file if target
+      when "both"
+        out = MultiIO.new(STDOUT, file) if file
+      else
+        out = STDOUT
+      end
+      out
+    end
+
+    def self.get_logger_level(level="fatal")
+      return nil unless level
+      logger_level = nil
+      case level.downcase
+      when "fatal"
+        logger_level = Logger::FATAL
+      when "error"
+        logger_level = Logger::ERROR
+      when "warn"
+        logger_level = Logger::WARN
+      when "info"
+        logger_level = Logger::INFO
+      when "debug"
+        logger_level = Logger::DEBUG
+      else
+        logger_level = Logger::FATAL
+      end
+      logger_level
+    end
+  end
+
+  class MultiIO
+    def initialize(*targets)
+      @targets = targets
+    end
+
+    def write(*args)
+      @targets.each {|t| t.write(*args)}
+    end
+
+    def close
+      @targets.each(&:close)
+    end
+  end
+
+end #Actory
+
+require 'rubygems'
+require 'msgpack/rpc'
+require 'parallel'
+require 'logger'
+Dir[File.join(File.dirname(__FILE__), "actory/*.rb")].each { |f| require_relative f }

data/lib/actory/error.rb

@@ -0,0 +1,7 @@
+module Actory
+  module Errors
+  end
+end
+
+require 'json'
+Dir[File.join(File.dirname(__FILE__), "errors/*.rb")].each { |f| require_relative f }

data/lib/actory/errors/generator.rb

@@ -0,0 +1,13 @@
+module Actory
+module Errors
+
+  class Generator
+
+    def json(level: nil, message: nil, backtrace: nil)
+      JSON.generate({:level => level.upcase, :message => message, :backtrace => backtrace})
+    end
+
+  end
+
+end #Errors
+end #Actory

data/lib/actory/receiver.rb

@@ -0,0 +1,17 @@
+module Actory
+module Receiver
+
+  class Base < Actory::Base
+    RECEIVER = YAML.load_file(File.expand_path(File.join(GLOBAL['config'], "/receiver.yml"), __FILE__))
+    @@logger = Logger.new(get_logger_output(RECEIVER['log']['type'], RECEIVER['log']['target']))
+    @@logger.level = get_logger_level(RECEIVER['log']['level'])
+  end
+
+end #Receiver
+end #Actory
+
+require 'msgpack/rpc/transport/udp'
+require 'facter'
+Dir[File.join(File.dirname(__FILE__), "receiver/*.rb")].each { |f| require_relative f }
+Dir[File.join(File.dirname(__FILE__), "receiver/plugin/*.rb")].each { |f| require_relative f }
+

data/lib/actory/receiver/event_handler.rb

@@ -0,0 +1,17 @@
+module Actory
+module Receiver
+
+class EventHandler < Base
+
+  def receive(method, arg=nil, results=[])
+    plugin = Actory::Receiver::Plugin.new
+    arg ? results << plugin.send(method, arg) : results << plugin.send(method)
+    return results
+  rescue => e
+    raise StandardError, e
+  end
+
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/auth.rb

@@ -0,0 +1,15 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def auth?(shared_key)
+    RECEIVER['shared_key'] == shared_key
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/example_fibonacci.rb

@@ -0,0 +1,17 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def fibonacci(num=10)
+    raise StandardError unless [Fixnum, String].include?(num.class)
+    num = num.to_i if num.class == String and num =~ /\A[0-9]+\z/
+    num <= 1 ? num : fibonacci(num - 1) + fibonacci(num - 2)
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/example_pi.rb

@@ -0,0 +1,20 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def pi(len=2)
+    raise StandardError unless [Fixnum, String].include?(len.class)
+    len = len.to_i if len.class == String and len =~ /\A[0-9]+\z/
+    b = 10 ** len
+    b2 = b << 1
+    pi = (len * 8 + 1).step(3, -2).inject(b) {|a, i| (i >> 1) * (a + b2) / i} - b
+    return "3.#{pi}"
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/example_prime.rb

@@ -0,0 +1,18 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+  require 'prime'
+
+  def prime(num=1)
+    raise StandardError unless [Fixnum, String].include?(num.class)
+    num = num.to_i if num.class == String and num =~ /\A[0-9]+\z/
+    Prime.each(num).to_a
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/processor_count.rb

@@ -0,0 +1,15 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def processor_count
+    return Parallel.processor_count
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/reload.rb

@@ -0,0 +1,16 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def reload
+    Dir[File.join(File.dirname(__FILE__), "./*.rb")].each { |f| load f }
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, e
+  end
+
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/plugin/system_info.rb

@@ -0,0 +1,31 @@
+module Actory
+module Receiver
+
+class Plugin < Base
+
+  def system_info
+    info = {
+      :processor => {
+        :count => Facter.value(:processorcount),
+        :physicalcount => Facter.value(:physicalprocessorcount),
+      },
+      :memory => {
+        :free => Facter.value(:memoryfree),
+        :size => Facter.value(:memorysize),
+      },
+      :swap => {
+        :free => Facter.value(:swapfree),
+        :size => Facter.value(:swapsize),
+      },
+      :timezone => Facter.value(:timezone),
+    }
+    return info
+  rescue => e
+    msg = Actory::Errors::Generator.new.json(level: "ERROR", message: e.message, backtrace: $@)
+    raise StandardError, msg
+  end
+
+end
+
+end #Receiver
+end #Actory

data/lib/actory/receiver/worker.rb

@@ -0,0 +1,43 @@
+module Actory
+module Receiver
+
+class Worker < Base
+  def initialize(protocol="tcp", target: nil)
+    protocol = RECEIVER['protocol'] if RECEIVER['protocol']
+    num = Parallel.processor_count
+    target ||= Actory::Receiver::EventHandler
+    Parallel.map(0..num, :in_processes => num) do |n|
+      @@logger.info "Starting Actory Receiver Worker ##{n + 1}/#{num} (PID = #{Process.pid}, PGROUP = #{Process.getpgrp}, protocol = #{protocol})"
+      is_retried = false
+      begin
+        worker = send(protocol, target, n)
+        Signal.trap(:TERM) { worker.stop }
+        Signal.trap(:INT) { worker.stop }
+        worker.run
+      rescue => e
+        @@logger.error(Actory::Errors::Generator.new.json(level: "error", message: e, backtrace: $@)) unless is_retried
+        is_retried = true
+        retry
+      end
+    end
+  end
+
+  private
+
+  def tcp(target, num)
+    worker = MessagePack::RPC::Server.new
+    worker.listen(RECEIVER['address'], RECEIVER['port'] + num, target.new)
+    worker
+  end
+
+  def udp(target, num)
+    address  = MessagePack::RPC::Address.new(RECEIVER['address'], RECEIVER['port'] + num)
+    listener = MessagePack::RPC::UDPServerTransport.new(address)
+    worker   = MessagePack::RPC::Server.new
+    worker.listen(listener, target.new)
+    worker
+  end
+end
+
+end #Receiver
+end #Actory

data/lib/actory/sender.rb

@@ -0,0 +1,14 @@
+module Actory
+module Sender
+
+  class Base < Actory::Base
+    SENDER = YAML.load_file(File.expand_path(File.join(GLOBAL['config'], "sender.yml"), __FILE__))
+    @@logger = Logger.new(get_logger_output(SENDER['log']['type'], SENDER['log']['target']))
+    @@logger.level = get_logger_level(SENDER['log']['level'])
+  end
+
+end #Sender
+end #Actory
+
+require 'progressbar'
+Dir[File.join(File.dirname(__FILE__), "sender/*.rb")].each { |f| require_relative f }

data/lib/actory/sender/dispatcher.rb

@@ -0,0 +1,172 @@
+module Actory
+module Sender
+
+class Dispatcher < Base
+  attr_accessor :actors, :trusted_hosts, :system_info, :receiver_count, :my_processor_count
+
+  def initialize(actors: [])
+    @actors = []
+    @trusted_hosts = []
+    @receiver_count = 0
+    @system_info = []
+    @my_processor_count = Parallel.processor_count
+    ret = initial_handshaking(actors)
+    raise StandardError if ret == 0
+    count = establish_connections
+    raise StandardError if count == 0
+  rescue => e
+    @@logger.error Actory::Errors::Generator.new.json(level: "error", message: "Initialization failed.", backtrace: $@)
+    exit 1
+  end
+
+  def message(method, args=[], results=[])
+    args = [nil] if args.empty?
+    assignment = assign_jobs(args)
+
+    pbar = ProgressBar.new(method, @receiver_count) if SENDER['show_progress']
+
+    results << Parallel.map(assignment, :in_processes => @receiver_count) do |arg, actor|
+      if SENDER['show_progress']
+        begin
+          pbar.set pbar.current + 1 if pbar.current <= @receiver_count
+        rescue
+        end
+      end
+
+      begin
+        actor.send("receive", "reload") if SENDER['reload_receiver_plugins']
+        res = actor.send("receive", method, arg)
+        sleep SENDER['get_interval']
+        ret = res.get
+        ret.flatten!
+        {actor.address.to_s => ret}
+      rescue => e
+        @@logger.warn Actory::Errors::Generator.new.json(level: "warn", message: "Something wrong with sending a message to #{actor.address}", backtrace: $@)
+        actor = change_actor(actor)
+        retry
+      end
+    end
+    results.flatten
+  end
+
+  private
+
+  def initial_handshaking(actors=[])
+    actors = SENDER['actors'] if actors.empty? and SENDER['actors'].nil? == false
+    actors.each do |actor|
+      next unless actor.class == String
+      actor = actor.gsub(/:/, " ").split
+      host = actor[0]
+      port = actor[1].to_i
+      @cli = MessagePack::RPC::Client.new(host, port)
+      @cli.timeout = SENDER['auth']['timeout']
+      ret = get_trusted_hosts(host)
+      next unless ret
+      @system_info << {:host => host, :system_info => get_system_info}
+      get_receiver_count
+    end
+  rescue => e
+    puts $@, e
+  end
+
+  def establish_connections
+    case SENDER['policy']
+    when "even"
+      establish_connections_evenly
+    when "random"
+      establish_connections_randomly(@receiver_count)
+    when "safe-random"
+      return 0 if @trusted_hosts.empty?
+      establish_connections_randomly(@my_processor_count / @trusted_hosts.count)
+    else
+      establish_connections_evenly
+    end
+    @actors.count
+  end
+
+  def establish_connections_randomly(num=0)
+    establish_connections_helper(num)
+  end
+
+  def establish_connections_evenly
+    return nil if @trusted_hosts.empty?
+    cores_per_host = @my_processor_count / @trusted_hosts.count
+    cores_per_host = 1 if cores_per_host <= 0
+    establish_connections_helper(cores_per_host)
+  end
+
+  def establish_connections_helper(num=0)
+    num.times do |n|
+      SENDER['actors'].each do |actor|
+        next unless actor.class == String
+        actor = actor.gsub(/:/, " ").split
+        host = actor[0]
+        next unless trusted_hosts.include?(host)
+        port = actor[1].to_i
+        cli = MessagePack::RPC::Client.new(host, port + n)
+        cli.timeout = SENDER['timeout']
+        @actors << cli
+      end
+    end
+    @@logger.debug @actors
+  end
+
+  def get_trusted_hosts(host)
+    res = @cli.send("receive", "auth?", SENDER['auth']['shared_key'])
+    res.get[0] ? @trusted_hosts << host : nil
+  rescue => e
+    @@logger.warn Actory::Errors::Generator.new.json(level: "warn", message: "#{__method__} failed with #{host}", backtrace: $@)
+    return nil
+  end
+
+  def get_system_info
+    res = @cli.send("receive", "system_info")
+    res.get[0]
+  end
+
+  def get_receiver_count
+    res = @cli.send("receive", "processor_count")
+    @receiver_count += res.get[0] if res.get[0]
+  end
+
+  def assign_jobs(args)
+    num = 0
+    params = {}
+    actors = @actors.sample(@my_processor_count)
+    args.each do |arg|
+      next if params.has_key?(arg)
+      num = 0 unless actors[num]
+      actor = actors[num]
+      num += 1
+      params.merge!(arg => actor)
+    end
+    @@logger.debug params
+    params
+  end
+
+  #def select_actors
+  #  actors = nil
+  #  case SENDER['policy']
+  #  when "even"
+  #    actors = @actors
+  #  when "random", "safe-random"
+  #    actors = @actors.sample(@my_processor_count)
+  #  else
+  #    actors = @actors
+  #  end
+  #  actors
+  #end
+
+  def change_actor(previous_actor)
+    new_actor = nil
+    loop do
+      new_actor = @actors.sample
+      break unless new_actor == previous_actor
+    end
+    new_actor
+  end
+
+end
+
+end #Sender
+end #Actory