smith 0.5.7

data/bin/agency

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+require 'pp'
+require 'pathname'
+
+root_path = Pathname.new(__FILE__).dirname.join('..').expand_path
+$:.unshift(root_path.join('lib'))
+
+require 'smith'
+require 'smith/acl_compiler'
+require 'smith/application/agency'
+
+module Smith
+  class AgencyRunner
+
+    include Logger
+
+    def initialize(root)
+
+      Smith.compile_acls
+      Smith.load_acls
+
+      @agency = Agency.new(:paths => Smith.agent_paths)
+
+      # Setup signal handlers to clean up.
+      %w{TERM INT QUIT}.each do |sig|
+        trap sig, proc { puts "Shutting down"; @agency.stop }
+      end
+    end
+
+    def run
+      Smith.start do
+
+        # This block is here so the that the shutdown hook added in
+        # Smith.start runs last. Yes I know this is leaky but that's how
+        # it is at the moment.
+
+        Smith.shutdown_hook do
+          # TODO put this as a command line option
+          if false
+            @compiler.clear_cache
+          end
+        end
+
+        logger.info { "Starting #{File.basename($0)}" }
+        @agency.setup_queues
+        @agency.start_monitoring
+      end
+    end
+  end
+end
+
+agency_runner = Smith::AgencyRunner.new(root_path)
+agency_runner.run

data/bin/smithctl

@@ -0,0 +1,102 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require 'smith'
+
+module Smith
+  class SmithControl
+
+    include Logger
+
+    def initialize
+      log_level(:info)
+    end
+
+    def send_command(command, args, &blk)
+      begin
+        send("#{Command.command_type(command)}_command", command, args, &blk)
+      rescue Smith::Command::UnkownCommandError => e
+        puts e.message
+        Smith.stop(true)
+      end
+    end
+
+    private
+
+    def smithctl_command(command, args, &blk)
+      Smith.channel.on_error do |ch,channel_close|
+        case channel_close.reply_code
+        when 404
+          puts "No such queue: #{extract_queue(channel_close.reply_text)}"
+        when 406
+          puts "Queue in use: #{extract_queue(channel_close.reply_text)}"
+        else
+          puts channel_close.reply_text
+          puts channel_close.reply_code
+        end
+      end
+
+      responder = Messaging::Responder.new
+      responder.callback do |v|
+        puts v if v
+        Smith.stop(true)
+      end
+
+      Command.run(command, args, :responder => responder)
+    end
+
+    def extract_queue(message)
+      match = /.*?'(.*?)'.*$/.match(message) #[1]
+      if match && match[1]
+        match[1].sub(/smith\./, '')
+      else
+        message
+      end
+    end
+
+    def agency_command(command, args, &blk)
+      Messaging::Sender.new('agency.control', :auto_delete => true, :durable => false, :persistent => true,  :strict => true).ready do |sender|
+
+        sender.timeout(8) { puts "Timeout. Is the agency still running"; Smith.stop(true) }
+
+        payload = ACL::Payload.new(:agency_command).content(:command => command, :args => args)
+
+        callback = proc do |sender|
+          sender.publish_and_receive(payload) do |r|
+            blk.call(r.payload)
+          end
+        end
+
+        errback = proc do puts "Agency not running"
+          Smith.stop(true)
+        end
+
+        sender.consumers?(callback, errback)
+      end
+    end
+  end
+end
+
+command = ARGV.shift
+args = ARGV
+
+command || (puts "usage #{File.basename($0)} <command> opts]"; exit 2)
+
+trap 'INT', proc { (Smith.running?) ? Smith.stop(true) : exit}
+
+Smith.load_acls
+
+control = Smith::SmithControl.new
+
+Smith.on_error do
+  Smith.stop(true)
+end
+
+Smith.start(:quiet => true) do
+  control.send_command(command, args) do |result|
+    puts result if result && !result.empty?
+    Smith.stop(true)
+  end
+end

data/lib/smith.rb

@@ -0,0 +1,237 @@
+# -*- encoding: utf-8 -*-
+require 'amqp'
+require 'tmpdir'
+require 'logging'
+require 'pathname'
+require 'fileutils'
+require 'optimism'
+require 'dm-core'
+require 'securerandom'
+require 'dm-yaml-adapter'
+require 'extlib/string'
+require 'extlib/inflection'
+require 'daemons/pidfile'
+
+require_relative 'smith/config'
+require_relative 'smith/logger'
+require_relative 'smith/acl_compiler'
+
+module Smith
+  include Logger
+
+  class << self
+
+    def channel
+      raise RuntimeError, "You must run this in a Smith.start block" if @channel.nil?
+      @channel
+    end
+
+    def on_error=(handler)
+      @handler = handler
+    end
+
+    def config
+      Config.get
+    end
+
+    def root_path
+      Pathname.new(File.dirname(__FILE__) + '/..').expand_path
+    end
+
+    def agent_paths
+      path_to_pathnames(config.agency.agent_path)
+    end
+
+    def acl_path
+      path_to_pathnames(config.agency.acl_path)
+    end
+
+    # Return the acl cache path. If it's not specified in the config
+    # generate a temporary path.
+    def acl_cache_path
+      @acl_cache_path ||= if Smith.config.agency._has_key?(:acl_cache_path)
+        Pathname.new(Smith.config.agency.acl_cache_path).tap { |path| check_path(path) }
+      else
+        cache_dir = Pathname.new(ENV['HOME']).join('.smith').join('acl')
+        if cache_dir.exist?
+          cache_dir
+        else
+          FileUtils.mkdir_p(cache_dir)
+          cache_dir
+        end
+      end
+    end
+
+    def compile_acls
+      @compiler = ACLCompiler.new
+      @compiler.compile
+    end
+
+    # Load all acls. This fucking horrible but for the time
+    # being it's how it's going to be. This will really start
+    # to be a problem when there are a lot of acls.
+    def load_acls
+      acl_cache_path.each_child do |acl_file|
+        logger.verbose { "Loading acl file: #{acl_file}" }
+        require acl_file
+      end
+    end
+
+    def running?
+      EM.reactor_running?
+    end
+
+    def on_error(&blk)
+      @on_error = blk
+    end
+
+    def start(opts={}, &block)
+      EM.epoll if EM.epoll?
+      EM.kqueue if EM.kqueue?
+      EM.set_descriptor_table_size(opts[:fdsize] || 1024)
+
+      connection_settings = config.amqp.broker._child.merge({
+        :on_tcp_connection_failure => method(:tcp_connection_failure_handler),
+        :on_possible_authentication_failure => method(:authentication_failure_handler)
+      })
+
+      AMQP.start(connection_settings) do |connection|
+        @connection = connection
+
+        connection.on_connection do
+          broker = connection.broker.properties
+          endpoint = connection.broker_endpoint
+          logger.debug { "Connected to: AMQP Broker: #{endpoint}, (#{broker['product']}/v#{broker['version']})" } unless opts[:quiet]
+        end
+
+        connection.on_tcp_connection_loss do |connection, settings|
+          EM.next_tick do
+            logger.error { "TCP connection error. Attempting restart" }
+            connection.reconnect
+          end
+        end
+
+        connection.after_recovery do
+          logger.info { "Connection to AMQP server restored" }
+        end
+
+        connection.on_error do |connection, reason|
+          case reason.reply_code
+          when 320
+            logger.warn { "AMQP server shutdown. Waiting." }
+          else
+            if @handler
+              @handler.call(connection, reason)
+            else
+              logger.error { "AMQP Server error: #{reason.reply_code}: #{reason.reply_text}" }
+            end
+          end
+        end
+
+        # This will be the last thing run by the reactor.
+        shutdown_hook { logger.debug { "Reactor Stopped" } }
+
+        AMQP::Channel.new(connection) do |channel,ok|
+          @channel = channel
+          # Set up QOS. If you do not do this then the subscribe in receive_message
+          # will get overwelmd and the whole thing will collapse in on itself.
+          channel.prefetch(1)
+
+          if @on_error
+            channel.on_error(&@on_error)
+          else
+            # Log the error and stop the agency when there are channel errors.
+            # TODO Add recovery instead of stopping the agency.
+            channel.on_error do |ch,channel_close|
+              logger.fatal { "Channel level exception: #{channel_close.reply_text}. Class id: #{channel_close.class_id}, Method id: #{channel_close.method_id}, Status code : #{channel_close.reply_code}" }
+              logger.fatal { "Agency is exiting" }
+              Smith.stop(true)
+            end
+          end
+
+          # Set up auto-recovery. This will ensure that the AMQP gem reconnects each
+          # channel and sets up the various exchanges & queues.
+          channel.auto_recovery = true
+
+          block.call
+        end
+      end
+    end
+
+    def shutdown_hook(&block)
+      EM.add_shutdown_hook(&block)
+    end
+
+    def stop(immediately=false, &blk)
+      shutdown_hook(&blk) if blk
+
+      if running?
+        if immediately
+          @connection.close { EM.stop_event_loop }
+        else
+          EM.add_timer(1) do
+            @connection.close { EM.stop_event_loop }
+          end
+        end
+      else
+        logger.fatal { "Eventmachine is not running, exiting with prejudice" }
+        exit!
+      end
+    end
+
+    private
+
+    def tcp_connection_failure_handler(settings)
+      # Only display the following settings.
+      s = settings.select { |k,v| ([:user, :pass, :vhost, :host, :port, :ssl].include?(k)) }
+
+      logger.fatal { "Cannot connect to the AMQP server." }
+      logger.fatal { "Is the server running and are the connection details correct?" }
+      logger.info { "Details:" }
+      s.each do |k,v|
+        logger.info { " Setting: %-7s%s" %  [k, v] }
+      end
+      EM.stop
+    end
+
+    def authentication_failure_handler(settings)
+      # Only display the following settings.
+      s = settings.select { |k,v| [:user, :pass, :vhost, :host].include?(k) }
+
+      logger.fatal { "Authenticaton failure." }
+      logger.info { "Details:" }
+      s.each do |k,v|
+        logger.info { " Setting: %-7s%s" %  [k, v] }
+      end
+      EM.stop
+    end
+
+    def path_to_pathnames(path)
+      path ||= []
+      path.split(':').map do |path|
+        p = Pathname.new(path)
+        ((p.absolute?) ? p : root_path.join(p)).tap { |path| check_path(path) }
+      end
+    end
+
+    def check_path(path)
+      logger.error("Path does not exist: #{path}") unless path.exist?
+    end
+  end
+end
+
+require_relative 'smith/cache'
+require_relative 'smith/agent'
+require_relative 'smith/agent_cache'
+require_relative 'smith/agent_process'
+require_relative 'smith/agent_monitoring'
+require_relative 'smith/command'
+require_relative 'smith/messaging/amqp_options'
+require_relative 'smith/messaging/queue_factory'
+require_relative 'smith/messaging/payload'
+require_relative 'smith/messaging/acl/default'
+require_relative 'smith/messaging/endpoint'
+require_relative 'smith/messaging/exceptions'
+require_relative 'smith/messaging/responder'
+require_relative 'smith/messaging/receiver'
+require_relative 'smith/messaging/sender'

data/lib/smith/acl_compiler.rb

@@ -0,0 +1,74 @@
+# -*- encoding: utf-8 -*-
+require 'pp'
+require 'protobuf/compiler/compiler'
+
+module Smith
+  class ACLCompiler
+
+    include Logger
+
+    def initialize(force=false)
+      # TODO Add the force code.
+      @cache_path = Smith.acl_cache_path
+    end
+
+    # Compile any protocol buffer files. This checks the timestamp
+    # to see if the file needs compiling.
+    def compile
+      logger.debug { "Protocol buffer cache path: #{@cache_path}" }
+      Smith.acl_path.each do |path|
+        results = {}
+        path_glob(path) do |p|
+          if should_compile?(p)
+            logger.info { "Compiling: #{p}" }
+            # TODO put some error handling here.
+            Protobuf::Compiler.compile(p.basename, p.dirname, @cache_path)
+          end
+        end
+      end
+      @cache_path
+    end
+
+    def cache_path
+      @cache_path.to_s
+    end
+
+    # Clears the Protocol Buffer cache. If acl_cache_path is
+    # specified in the config then the directory itself won't be removed
+    # but if it's not specified and a temporary directory was created then
+    # the directory is removed as well.
+    def clear_cache
+      logger.info { "Clearing the Protocol Buffer cache: #{Smith.acl_cache_path}" }
+
+      Pathname.glob(@cache_path.join("*")).each do |path|
+        path.unlink
+      end
+
+      unless Smith.config.agency._has_key?(:acl_cache_path)
+        @cache_path.rmdir
+      end
+    end
+
+    private
+
+    # Returns true if the .proto file is newer that the .pb.rb file
+    def should_compile?(file)
+      cached_file = @cache_path.join(file.basename).sub_ext(".pb.rb")
+      if cached_file.exist?
+        if file.mtime > cached_file.mtime
+          true
+        else
+          false
+        end
+      else
+        true
+      end
+    end
+
+    def path_glob(path)
+      Pathname.glob("#{path.join("*.proto")}").map do |acl|
+        yield acl.realpath
+      end
+    end
+  end
+end