smith 0.5.13.1 → 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 95209344632520fac5a790a8efa4cfd91a52ddb2
+  data.tar.gz: 562bb8a1c8bd5371fdf3d74765f8c069b88f0fa0
+SHA512:
+  metadata.gz: 2a8202bf141e34265c7254ba6dfef92d7798b8d6bd9ff7b0bc903060a27d11d86d38ba8f1b1be7aede80932f10eee31f1fd21cdb1d22655e1ffc212b039543c2
+  data.tar.gz: a5e5d205a49c1fd10ada42bf317fe17bde5cf12d375746133b909ea87892ecd7b35462f6b6c9cb8e8267872503799a8ee99ab6acaa921cac287221369d383718

data/bin/agency

@@ -1,11 +1,13 @@
 #!/usr/bin/env ruby
 # -*- encoding: utf-8 -*-
 
-$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'pathname'
+
+$:.unshift(Pathname.new(__FILE__).dirname.parent.join('lib').expand_path)
 
-require 'pp'
 require 'smith'
 require 'smith/daemon'
+require 'smith/agent_cache'
 require 'smith/acl_compiler'
 require 'smith/application/agency'
 
@@ -21,16 +23,19 @@ module Smith
       @options = opts
       options_check
 
-      @daemon = Daemon.new(AGENCY_NAME, opts[:daemon], opts[:pid_dir])
-      @agency = Agency.new(:paths => Smith.agent_paths)
+      @daemon = Daemon.new(add_vhost(AGENCY_NAME), opts[:daemon], opts[:pid_dir])
+      @agency = Agency.new
 
       Smith.shutdown_hook do
-        puts "Shutting down"
+        logger.info { "Shutting down" }
       end
 
       # Setup signal handlers to clean up.
       %w{TERM INT QUIT}.each do |sig|
-        trap sig, proc { @agency.stop }
+        trap sig, proc {
+          logger.info { "Shutting down" }
+          @agency.stop
+        }
       end
     end
 
@@ -38,10 +43,12 @@ module Smith
       if @daemon.running?
         logger.fatal { "The agency is alredy running" }
       else
+        logger.info { "Using config file: #{Smith.config_path}" }
+        require 'smith/messaging/acl_type_cache'
+
         @daemon.daemonise
 
         Smith.compile_acls
-        Smith.load_acls
 
         Smith.start do
           # This block is here so the that the shutdown hook added in
@@ -66,6 +73,11 @@ module Smith
         puts "Logger set to stdout and daemonise is true. Log messages will be sent to /dev/null."
       end
     end
+
+    def add_vhost(name)
+      suffix = Pathname.new(Smith.config.amqp.broker.vhost).basename
+      "%s.%s" % [name, (suffix.root?) ? 'root' : suffix.to_s]
+    end
   end
 end
 

data/bin/pry-smith

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# -*- encoding: utf-8 -*-
+
+require 'pathname'
+$:.unshift(Pathname.new(__FILE__).dirname.parent.join('lib').expand_path)
+
+require 'pry'
+require 'smith'
+
+Smith.compile_acls
+Pry::CLI.parse_options

data/bin/smithctl

@@ -5,20 +5,22 @@
   trap sig, proc { exit }
 end
 
-
-$:.unshift(File.dirname(__FILE__) + '/../lib')
+require 'pathname'
+$:.unshift(Pathname.new(__FILE__).dirname.parent.join('lib').expand_path)
 
 require 'smith'
+require 'smith/command'
+
+require 'trollop'
 
 include Smith::Logger
 
 module Smith
   class SmithControl
 
-
     def initialize(options={})
-      log_level((options[:log_level_given]) ? options[:log_level].to_sym : :info)
-      @timeout = (options[:timeout_given]) ? options[:timeout] : Smith.config.agency.timeout
+      log_level((options[:log_level_given]) ? options[:log_level].to_sym : :warn)
+      @timeout = (options[:timeout_given]) ? options[:timeout] : Smith.config.smith.timeout
     end
 
     def send_command(command, args, &blk)
@@ -37,23 +39,19 @@ module Smith
 
     def agency_command(command, args, &blk)
       Messaging::Sender.new(QueueDefinitions::Agency_control) do |sender|
+        sender.on_timeout(5) { |message_id| blk.call("Timeout. Is the agency still running?") }
 
-        timeout = Messaging::Timeout.new(5) { |message_id| blk.call("Timeout. Is the agency still running?") }
-
-        sender.on_reply(:timeout => timeout) do |reply_payload, r|
-          r.ack
+        sender.on_reply(:auto_ack => true) do |reply_payload, r|
           blk.call(reply_payload[:response])
         end
 
-        # callback = proc do |sender|
-          sender.publish(Smith::ACL::Factory.create(:agency_command, :command => command, :args => args))
-        # end
-
-        # errback = proc do
-        #   blk.call("Agency not running.")
-        # end
-
-        # sender.consumers?(callback, errback)
+        sender.consumer_count do |count|
+          if count > 0
+            sender.publish(Smith::ACL::AgencyCommand.new(:command => command, :args => args))
+          else
+            blk.call("Agency not running.")
+          end
+        end
       end
     end
   end
@@ -84,14 +82,14 @@ Usage:
     end
   end
 
-  command = ARGV.shift
-  args = ARGV
+  command = ARGV.shift.dup
+  args = ARGV.map {|a| a.dup}
 
   %w{TERM INT QUIT}.each do |sig|
     trap sig, proc { (Smith.running?) ? Smith.stop(true) : exit}
   end
 
-  Smith.load_acls
+  Smith.compile_acls
 
   control = SmithControl.new(opts)
 

data/lib/smith/acl_compiler.rb

@@ -1,89 +1,134 @@
 # -*- encoding: utf-8 -*-
-require 'pp'
-require 'protobuf/compiler/compiler'
-
-# There doesn't seem to be a way to turn of printing. The code is generally run
-# like this: log_writing unless silent but there is no way of setting silent!
-# So monkey patch it.
-class Protobuf::Visitor::Base
-  def log_writing(filename, message="writing..."); end
-end
+require 'ffi'
+require "tempfile"
+require 'ruby_parser'
+require 'smith/messaging/acl_type_cache'
 
 module Smith
   class ACLCompiler
 
     include Logger
+    extend FFI::Library
 
-    def initialize(force=false)
-      # TODO Add the force code.
-      @cache_path = Smith.acl_cache_path
+    def self.find_so
+      $:.map{|p| Pathname.new(p).join("ruby_generator.so")}.detect{|so| so.exist? }
+    end
+
+    ffi_lib(find_so)
+    attach_function(:_rprotoc_extern, [:int, :pointer], :int32)
+
+    def initialize
+      @acl_type_cache = AclTypeCache.instance
+      @acl_parser = ACLParser.new
     end
 
-    # Compile any protocol buffer files. This checks the timestamp to see if
-    # the file needs compiling. This is done in a subprocess to stop the agency
-    # from dying. I think it's a problem with the class being loaded twice but
-    # I don't actually know that so this could be considered a bit brute force.
     def compile
-      Process.fork do
-        logger.debug { "Protocol buffer cache path: #{@cache_path}" }
-        Smith.acl_path.each do |path|
-          results = {}
-          path_glob(path) do |p|
-            if should_compile?(p)
-              begin
-                logger.info { "Compiling: #{p}" }
-                Protobuf::Compiler.compile(p.basename, p.dirname, @cache_path)
-              rescue Racc::ParseError => e
-                logger.error { "Cannot parse: #{p}" }
-              end
-            end
-          end
+      $LOAD_PATH << Smith.acl_cache_path
+
+      Smith.acl_path.each do |path|
+        $LOAD_PATH << path
+
+        acls_files = path_glob(path)
+        out_of_date_acls = path_glob(path).select { |p| should_compile?(p) }
+        if out_of_date_acls.size > 0
+          compile_on_path(path, acls_files, out_of_date_acls)
+        end
+
+        acls_files.each do |acl_file|
+          acl_class_path = acl_compiled_path(acl_file)
+          load_acl(acl_class_path)
+          add_to_type_cache(acl_class_path)
         end
       end
-      Process.wait
-      @cache_path
     end
 
-    def cache_path
-      @cache_path.to_s
-    end
+    private
 
-    # Clears the Protocol Buffer cache. If acl_cache_path is
-    # specified in the config then the directory itself won't be removed
-    # but if it's not specified and a temporary directory was created then
-    # the directory is removed as well.
-    def clear_cache
-      logger.info { "Clearing the Protocol Buffer cache: #{Smith.acl_cache_path}" }
+    def compile_on_path(path, acls, out_of_date_acls)
+      out_of_date_acls.each { |acl| logger.debug("Compiling acl: #{path.join(acl)}") }
 
-      Pathname.glob(@cache_path.join("*")).each do |path|
-        path.unlink
-      end
+      unless acls.empty?
+        Dir.chdir(path) do
+          begin
+            GC.disable
+
+            args = ["rprotoc", "--ruby_out", Smith.acl_cache_path, "--proto_path", path].map {|a| ::FFI::MemoryPointer.from_string(a.to_s.dup) }
+
+            ffi_acls = acls.map do |acl|
+              FFI::MemoryPointer.from_string(acl.to_s.dup)
+            end
+            ffi_acls << nil
+
+            args += ffi_acls
+            argv = FFI::MemoryPointer.new(:pointer, args.size)
+
+            args.each_with_index { |p, index| argv[index].put_pointer(0, p) }
 
-      unless Smith.config.agency._has_key?(:acl_cache_path)
-        @cache_path.rmdir
+            errors = capture_stderr do
+              self._rprotoc_extern(args.compact.size, argv)
+            end.split("\n")
+
+            errors.each do |error|
+              logger.fatal { "Cannot compile ACLs: #{error}" }
+              raise RuntimeError, error
+            end
+          ensure
+            GC.enable
+          end
+        end
       end
     end
 
-    private
-
     # Returns true if the .proto file is newer that the .pb.rb file
     def should_compile?(file)
-      cached_file = @cache_path.join(file.basename).sub_ext(".pb.rb")
+      cached_file = Smith.acl_cache_path.join(file.basename).sub_ext(".pb.rb")
       if cached_file.exist?
-        if file.mtime > cached_file.mtime
-          true
-        else
-          false
-        end
+        file.mtime > cached_file.mtime
       else
         true
       end
     end
 
     def path_glob(path)
-      Pathname.glob("#{path.join("*.proto")}").map do |acl|
-        yield acl.realpath
+      Pathname.glob(path.join("*.proto")).map { |acl| acl.realpath }
+    end
+
+    # This is not idea but I really don't know how else to do it. I cannot use
+    # $stderr = StringIO.new as this only seems to capture STDERR if you
+    # explicitly write to STDERR. In my case it's an ffi library call that's
+    # writing to STDERR.
+    def capture_stderr
+      org_stderr = STDERR.dup
+      begin
+        tmp = Tempfile.new('')
+        tmp.sync = true
+        STDERR.reopen(tmp)
+        yield
+        File.read(tmp.path)
+      ensure
+        STDERR.reopen(org_stderr)
       end
     end
+
+    def load_acl(path)
+      logger.verbose { "Loading ACL: #{path}" }
+      require path
+    end
+
+    def add_to_type_cache(path)
+      acl_class = File.read(path)
+      @acl_parser.go(acl_class)
+      @acl_parser.fully_qualified_classes.each do |clazz|
+        @acl_type_cache.add(to_class(clazz))
+      end
+    end
+
+    def to_class(type_as_array)
+      type_as_array.inject(Kernel) { |acc, t| acc.const_get(t) }
+    end
+
+    def acl_compiled_path(path)
+      "#{Smith.acl_cache_path.join(path.basename('.proto'))}.pb.rb"
+    end
   end
 end

data/lib/smith/acl_parser.rb

@@ -0,0 +1,75 @@
+# -*- encoding: utf-8 -*-
+
+require 'set'
+require 'sexp_processor'
+require 'ruby_parser'
+require 'optparse'
+require 'timeout'
+
+module Smith
+  class ACLParser < SexpProcessor
+
+    def initialize
+      super()
+      @class_stack         = []
+      self.auto_shift_type = true
+      self.reset
+    end
+
+    def go(ruby)
+      @parser = RubyParser.new
+      process(@parser.process(ruby))
+    end
+
+    # Adds name to the class stack, for the duration of the block
+    def in_class(name)
+      @class_stack.unshift(name)
+      yield
+      @class_stack.shift
+    end
+
+    # Returns the first class in the list, or :main
+    def class_name
+      if @class_stack.any?
+        @class_stack.reverse
+      else
+        :main
+      end
+    end
+
+    # Process each element of #exp in turn.
+    def process_until_empty(exp)
+      process(exp.shift) until exp.empty?
+    end
+
+    def fully_qualified_classes
+      @classes.delete(:main)
+      @classes.inject([]) do |a, class_method|
+        a.tap do |acc|
+          acc << class_method
+        end
+      end
+    end
+
+    # Reset @classes data
+    def reset
+      @classes = Set.new
+    end
+
+    # Process Class method
+    def process_class(exp)
+      in_class(exp.shift) do
+        process_until_empty exp
+        @classes << class_name
+      end
+      s()
+    end
+
+    def process_module(exp)
+      in_class exp.shift do
+        process_until_empty exp
+      end
+      s()
+    end
+  end
+end

data/lib/smith/agent.rb

@@ -6,11 +6,12 @@ module Smith
     include Logger
     include Smith::ObjectCount
 
-    attr_reader :factory, :name, :pid
+    attr_reader :name, :pid, :uuid
 
-    def initialize(options={})
+    def initialize(uuid)
       @name = self.class.to_s
       @pid = $$
+      @uuid = uuid
 
       @factory = QueueFactory.new
 
@@ -67,16 +68,9 @@ module Smith
       @on_running = blk
     end
 
-    # Override this method to implement your own agent. You can use task but this may
-    # go away in the future. This method must not block.
+    # Override this method to implement your own agent.
     def run
-      raise ArgumentError, "You need to call Agent.task(&block)" if @@task.nil?
-
-      logger.debug { "Setting up default queue: #{default_queue_name}" }
-
-      subscribe(default_queue_name, :auto_delete => false) do |r|
-        @@task.call(r.payload)
-      end
+      raise ArgumentError, "You must override this method"
     end
 
     def install_signal_handler(signal, position=:end, &blk)
@@ -107,16 +101,14 @@ module Smith
     end
 
     class << self
-      def task(opts={}, &blk)
-        @@task = blk
-      end
-
       # Options supported:
       # :monitor,   the agency will monitor the agent & if dies restart.
       # :singleton, only every have one agent. If this is set to false
       #             multiple agents are allowed.
       def options(opts)
-        Smith.config.agent._merge!(opts)
+        opts.each do |k, v|
+          Smith.config.agent.send("#{k}=", v)
+        end
       end
     end
 
@@ -128,9 +120,9 @@ module Smith
     end
 
     def setup_control_queue
-      logger.debug { "Setting up control queue: #{control_queue_name}" }
+      logger.debug { "Setting up control queue: #{control_queue_def.denormalise}" }
 
-      Messaging::Receiver.new(control_queue_name, :durable => false, :auto_delete => true) do |receiver|
+      Messaging::Receiver.new(control_queue_def) do |receiver|
         receiver.subscribe do |payload|
           logger.debug { "Command received on agent control queue: #{payload.command} #{payload.options}" }
 
@@ -142,7 +134,7 @@ module Smith
           when 'log_level'
             begin
               level = payload.options.first
-              logger.info { "Setting log level to #{level} for: #{name}" }
+              logger.info { "Setting log level to #{level} for: #{name} (#{uuid})" }
               log_level(level)
             rescue ArgumentError => e
               logger.error { "Incorrect log level: #{level}" }
@@ -155,20 +147,18 @@ module Smith
     end
 
     def setup_stats_queue
-      puts "setting stats"
-      # instantiate this queue without using the factory so it doesn't show
-      # up in the stats.
       Messaging::Sender.new(QueueDefinitions::Agent_stats) do |stats_queue|
         EventMachine.add_periodic_timer(2) do
           stats_queue.number_of_consumers do |consumers|
             if consumers > 0
-              payload = ACL::Factory.create(:agent_stats) do |p|
+              payload = ACL::AgentStats.new.tap do |p|
+                p.uuid = uuid
                 p.agent_name = self.name
                 p.pid = self.pid
                 p.rss = (File.read("/proc/#{pid}/statm").split[1].to_i * 4) / 1024 # This assumes the page size is 4K & is MB
                 p.up_time = (Time.now - @start_time).to_i
-                factory.each_queue do |q|
-                  p.queues << ACL::Factory.create('agent_stats::queue_stats', :name => q.denormalised_queue_name, :type => q.class.to_s, :length => q.counter)
+                queues.each_queue do |q|
+                  p.queues << ACL::AgentStats::QueueStats.new(:name => q.name, :type => q.class.to_s, :length => q.counter)
                 end
               end
 
@@ -180,24 +170,23 @@ module Smith
     end
 
     def acknowledge_start(&blk)
-      Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |ack_start_queue|
-        payload = ACL::Factory.create(:agent_lifecycle) do |p|
-          p.state = 'acknowledge_start'
+      Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |queue|
+        payload = ACL::AgentAcknowledgeStart.new.tap do |p|
+          p.uuid = uuid
           p.pid = $$
-          p.name = self.class.to_s
-          p.metadata = Smith.config.agent.metadata
-          p.monitor = Smith.config.agent.monitor
           p.singleton = Smith.config.agent.singleton
           p.started_at = Time.now.to_i
+          p.metadata = Smith.config.agent.metadata
+          p.monitor = Smith.config.agent.monitor
         end
-        ack_start_queue.publish(payload, &blk)
+        queue.publish(payload)
       end
     end
 
     def acknowledge_stop(&blk)
-      Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |ack_stop_queue|
+      Messaging::Sender.new(QueueDefinitions::Agent_lifecycle) do |queue|
         message = {:state => 'acknowledge_stop', :pid => $$, :name => self.class.to_s}
-        ack_stop_queue.publish(ACL::Factory.create(:agent_lifecycle, message), &blk)
+        queue.publish(ACL::AgentAcknowledgeStop.new(:uuid => uuid), &blk)
       end
     end
 
@@ -205,10 +194,10 @@ module Smith
       if Smith.config.agent.monitor
         EventMachine::add_periodic_timer(1) do
           Messaging::Sender.new(QueueDefinitions::Agent_keepalive) do |queue|
-            message = {:name => self.class.to_s, :pid => $$, :time => Time.now.to_i}
-            keep_alive_queue.consumers do |consumers|
+            message = {:name => self.class.to_s, :uuid => uuid, :time => Time.now.to_i}
+            queue.consumers do |consumers|
               if consumers > 0
-                keep_alive_queue.publish(ACL::Factory.create(:agent_keepalive, message))
+                queue.publish(ACL::AgentKeepalive, message)
               end
             end
           end
@@ -222,12 +211,8 @@ module Smith
       @factory
     end
 
-    def control_queue_name
-      "#{default_queue_name}.control"
-    end
-
-    def default_queue_name
-      "agent.#{name.sub(/Agent$/, '').snake_case}"
+    def control_queue_def
+      @control_queue_def ||= QueueDefinitions::Agent_control.call(uuid)
     end
   end
 end

data/lib/smith/agent_cache.rb

@@ -1,40 +1,66 @@
 # -*- encoding: utf-8 -*-
+require 'tdb'
+require 'securerandom'
+
 module Smith
-  class AgentCache < Cache
+  class AgentCache
+
+    include Enumerable
 
     attr_accessor :path
 
     def initialize(opts={})
-      super()
-      @paths = opts[:paths]
+      # @db = LevelDB::DB.make(Smith.cache_path.join('agent_state').to_s, :error_if_exists => false, :create_if_missing => true)
+      @db = TDB.new(Smith.cache_path.join('agent_state.tdb').to_s)
+    end
 
-      operator ->(agent_name, options={}) { AgentProcess.first(:name => agent_name) || AgentProcess.new(:name => agent_name, :path => agent_path(agent_name)) }
+    def create(name)
+      AgentProcess.new(@db, :name => name, :uuid => SecureRandom.uuid)
+    end
 
-      populate
+    def alive?(uuid)
+      (@db.include?(uuid)) ? instantiate(@db[uuid]).alive? : false
     end
 
-    def alive?(name)
-      (exist?(name)) ? entry(name).alive? : false
+    def exist?(uuid)
+      @db.include?(uuid)
     end
 
+    def find_by_name(*names)
+      inject([]) do |a, agent|
+        a.tap do |acc|
+          names.flatten.each do |name|
+            acc << agent if name == agent.name
+          end
+        end
+      end
+    end
+
+      # select {|a| a.name == name.to_s }
+    # end
+
     def state(state)
       select {|a| a.state == state.to_s }
     end
 
-    alias names :entries
-    alias :[] :entry
+    def delete(uuid)
+      @db.delete(uuid)
+    end
 
-    private
+    def entry(uuid)
+      (uuid) ? instantiate(@db[uuid]) : nil
+    end
+
+    alias :[] :entry
 
-    # When we start load any new data from the db.
-    def populate
-      AgentProcess.all.each { |a| update(a.name, a) }
+    def each(&blk)
+      @db.each {|k,v| blk.call(instantiate(v)) }
     end
 
-    def agent_path(agent_name)
-      @paths.detect do |path|
-        Pathname.new(path).join("#{agent_name.snake_case}.rb").exist?
-      end
+    private
+
+    def instantiate(state)
+      (state) ? AgentProcess.new(@db, state) : nil
     end
   end
 end

data/lib/smith/agent_monitoring.rb

@@ -38,7 +38,7 @@ module Smith
               logger.info { "Restarting dead agent: #{agent_process.name}" }
               Messaging::Sender.new(QueueDefinitions::Agency_control) do |sender|
                 sender.on_reply { |p, r| logger.debug { "Agent restart message acknowledged: #{agent_process.name}" } }
-                sender.publish(ACL::Factory.create(:agency_command).content(:command => 'start', :args => [agent_process.name]))
+                sender.publish(ACL::AgencyCommand.new(:command => 'start', :args => [agent_process.name]))
               end
             when 'unknown'
               logger.info { "Agent is in an unknown state: #{agent_process.name}" }