emissary 1.3.0

data/lib/emissary/agent/sshkeys.rb

@@ -0,0 +1,152 @@
+#   Copyright 2010 The New York Times
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+require 'digest/md5'
+require 'base64'
+require 'etc'
+
+module Emissary
+  class Agent::Sshkeys < Agent
+    AUTH_KEY_FILE = '.ssh/authorized_keys'
+
+    attr_reader :user
+    
+    def valid_methods
+      [ :add, :delete, :update ]
+    end
+    
+    def post_init
+      begin
+        @user = Etc.getpwnam(args.shift)
+      rescue ArgumentError => e
+        if e.message =~ /can't find user/
+          raise "User '#{args.first}' does not exist on this system"
+        else
+          raise "Unhandled error attempting to retrieve data on user [#{args.first}]: #{e.message}"
+        end
+      end
+    end
+    
+    def add pubkey
+      raise "Missing 'key_uri' argument - can't download key!" if pubkey.nil?
+
+      pubkey_name = Digest::MD5.hexdigest(pubkey.split(/\s+/).join(' ').chomp)
+
+      begin
+        keys = get_keys(user)
+        if not keys.has_key?(pubkey_name)
+          keys[pubkey_name] = pubkey
+          write_keys(user, keys)
+          result = "Successfully added key [#{pubkey_name}] to user [#{user.name}]"
+        else
+          result = "Could not add key [#{pubkey_name}] to user [#{user.name}] - key already exists!"
+        end
+      rescue Exception => e
+        raise Exception, 'Possibly unable to add user key - error was: ' + e.message, caller
+      end
+
+      return result
+    end
+    
+    def delete pubkey
+      return 'No authorized_keys file - nothing changed' if not File.exists?(File.join(user.dir, AUTH_KEY_FILE))
+  
+      keyname = Digest::MD5.hexdigest(pubkey)
+      begin
+        keys = get_keys(user)
+        if keys.has_key?(keyname)
+          keys.delete(keyname)
+          write_keys(user, keys)
+          result = "Successfully removed key [#{keyname}] from user [#{user.name}]"
+        else
+          result = "Could not remove key [#{keyname}] from user [#{user.name}] - key not there!"
+        end
+      rescue Exception => e
+        raise "Possibly unable to remove key #{keyname} for user #{user.name} - error was: #{e.message}"
+      end
+  
+      return result
+    end
+
+    def update oldkey, newkey
+      return 'Not implemented'
+    end
+    
+    private
+    
+    def write_keys(user, keys)
+      auth_file_do(user, 'w') do |fp|
+        fp << keys.values.join("\n")
+      end
+    end
+  
+    def get_keys(user)
+      ::Emissary.logger.debug "retreiving ssh keys from file #{File.join(user.dir, AUTH_KEY_FILE)}"
+  
+      keys = {}
+      auth_file_do(user, 'r') do |fp|
+        fp.readlines.each do |line|
+          keys[Digest::MD5.hexdigest(line.chomp)] = line.chomp
+        end
+      end
+  
+      return keys
+    end
+  
+    def auth_file_setup(user)
+      user_auth_file = File.join(user.dir, AUTH_KEY_FILE)
+      user_ssh_dir   = File.dirname(user_auth_file)
+  
+      begin
+        if not File.directory?(user_ssh_dir)
+          Dir.mkdir(user_ssh_dir)
+          File.open(user_ssh_dir) do |f|
+            f.chown(user.uid, user.gid)
+            f.chmod(0700)
+          end
+        end
+        if not File.exists?(user_auth_file)
+          File.open(user_auth_file, 'a') do |f|
+            f.chown(user.uid, user.gid)
+            f.chmod(0600)
+          end
+        end
+      rescue Exception => e
+        raise "Error creating #{user_auth_file} -- #{e.message}"
+      end
+  
+      return user_auth_file
+    end
+  
+    def auth_file_do(user, mode = 'r', &block)
+      begin
+        auth_file = auth_file_setup(user)
+        File.open(auth_file, mode) do |f|
+          f.flock File::LOCK_EX
+          yield(f)
+          f.flock File::LOCK_UN
+        end
+      rescue Exception => e
+        case mode
+          when 'r': mode = 'reading from'
+          when 'a': mode = 'appending to'
+          when 'w': mode = 'writing to'
+          else mode = "unknown operation #{mode} for File.open() on "
+        end
+        raise "Error #{mode} file '#{auth_file}' --  #{e.message}"
+      end
+    end
+  end
+end

data/lib/emissary/agent/stats.rb

@@ -0,0 +1,96 @@
+#   Copyright 2010 The New York Times
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+module Emissary
+  class Agent::Stats < Agent    
+    STATISTIC_TYPES = [ :cpu, :network, :disk ]
+
+    begin
+      require 'sys/cpu'
+    rescue LoadError
+      STATISTIC_TYPES.delete(:cpu)
+      ::Emissary.logger.warning "Ruby Gem 'sys-cpu' doesn't appear to be present - removing statistic gather for cpu."
+    end
+    
+    begin
+      require 'ifconfig' 
+    rescue LoadError
+      STATISTIC_TYPES.delete(:network)
+      ::Emissary.logger.warning "Ruby Gem 'ifconfig' doesn't appear to be present - removing statistic gather for network."
+    end
+
+    def valid_methods
+      [ :gather ]
+    end
+    
+    def gather
+      message.recipient = "#{config[:stats][:queue_base]}:#{message.exchange_type.to_s}"
+      message.args = STATISTIC_TYPES.inject([]) do |args, type|
+        unless (data = self.__send__(type)).nil?
+          args << { type => data }
+        end
+        args
+      end
+
+      throw :skip_implicit_response unless not message.args.empty?
+      return message
+    end
+
+    def disk
+      @cmd = "/usr/bin/env df -B K -P -T -x devfs -x tmpfs | /usr/bin/env tail -n +2"
+      
+      data = IO.popen(@cmd){ |f| f.readlines }.collect { |l| l.split(/\s+/) }
+      data.inject([]) { |data,line|
+        device = Hash[[:device, :type, :size, :used, :avail, :percent, :mount].zip(line.collect!{|v| v =~ /^\d+/ ? v[/^(\d+)/].to_i : v })]
+        
+        ::Emissary.logger.notice("[statistics] Disk#%s: type:%s mount:%s size:%d used:%d in-use:%d%%",
+          device[:device], device[:type], device[:mount], device[:size], device[:used], device[:percent]
+        )
+        
+        data << device
+      }
+    end
+    
+    def cpu
+      load_average = Sys::CPU.load_avg
+      ::Emissary.logger.notice "[statistics] CPU: #{load_average.join ', '}"
+      load_average
+    end
+    
+    def network
+      interfaces = (ifconfig = IfconfigWrapper.new.parse).interfaces.inject([]) do |interfaces, name|
+        interfaces << (interface = {
+          :name => name,
+          :tx   => ifconfig[name].tx.symbolize,
+          :rx   => ifconfig[name].rx.symbolize,
+          :up   => ifconfig[name].status,
+          :ips  => ifconfig[name].addresses('inet').collect { |ip| ip.to_s }
+        })
+        
+        ::Emissary.logger.notice("[statistics] Network#%s: state:%s tx:%d rx:%d inet:%s",
+          name,
+          (interface[:up] ? 'up' : 'down'),
+          interface[:tx][:bytes],
+          interface[:rx][:bytes],
+          interface[:ips].join(',')
+        ) unless interface.try(:[], :tx).nil?
+        
+        interfaces
+      end
+      
+      return interfaces
+    end
+  end
+end

data/lib/emissary/agent/test.rb

@@ -0,0 +1,40 @@
+#   Copyright 2010 The New York Times
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+module Emissary
+  class Agent::Test < Agent
+    def valid_methods
+      [:test_raise]
+    end
+    
+    def test_raise klass, *args
+      ::Emissary.logger.debug "TEST AGENT: #test(#{klass}, #{args.inspect})"
+
+      exception = nil
+      begin
+        e_klass = ::Emissary.klass_const(klass)
+        unless not e_klass.try(:new).try(:is_a?, Exception)
+          raise e_klass, *args
+        else
+          raise Exception, "#{e_klass.name.to_s rescue e_klass.to_s} is not a valid exception name!"
+        end
+      rescue Exception => e
+        exception = e
+      end
+
+      message.error exception
+    end
+  end
+end

data/lib/emissary/config.rb

@@ -0,0 +1,231 @@
+#   Copyright 2010 The New York Times
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+#
+require 'inifile'
+
+#
+# This class represents the INI file and can be used to parse INI files.
+# Derived from IniFile gem, found on http://rubyforge.org/projects/inifile/
+#
+module Emissary
+  class ConfigParseError < ::Emissary::Error
+    def initialize(message)
+      super(Exception, message)
+    end
+  end
+  
+  class ConfigValidationError < ::Emissary::Error
+    def initialize(message)
+      super(Exception, message)
+    end
+  end
+
+  class ConfigFile < IniFile
+
+    attr_reader :ini
+    def initialize( filename, opts = {} )
+      @line_number = 0
+      @fn = filename
+      @comment = opts[:comment] || '#'
+      @param = opts[:parameter] || '='
+      @debug = !!opts[:debug]
+      @ini = Hash.new {|h,k| h[k] = Hash.new}
+  
+      @rgxp_comment    = /^\s*$|^\s*[#{@comment}]/
+      @rgxp_section    = /^\s*\[([^\]]+)\]/
+      @rgxp_param      = /^([^#{@param}]+)#{@param}(.*)$/
+  
+      @rgxp_dict_start = /^([^#{@param}]+)#{@param}\s*\{\s*$/
+      @rgxp_dict_stop  = /^\s*\}\s*$/
+      @dict_stack      = []
+  
+      @rgxp_list_start = /^([^#{@param}]+)#{@param}\s*\[\s*$/
+      @rgxp_list_line  = /^([^#{@param}]+)#{@param}\s*\[\s*([^\]]+)\]\s*$/
+      @rgxp_list_stop  = /^\s*\]\s*$/
+      @list_items      = []
+      @in_list_name    = nil
+      
+      super filename, opts
+      
+      yield self if block_given?
+    end
+  
+    #
+    # call-seq:
+    #    ini_file[section]
+    #
+    # Get the hash of parameter/value pairs for the given _section_.
+    #
+    def []( section )
+      return nil if section.nil?
+      @ini[section.to_sym]   
+    end   
+           
+    #
+    # call-seq:
+    #    has_section?( section )
+    #
+    # Returns +true+ if the named _section_ exists in the INI file.
+    #
+    def has_section?( section )
+      @ini.has_key? section.to_sym
+    end
+  
+    #
+    # call-seq:
+    #    parse
+    #
+    # Loops over each line of the file, passing it off to the parse_line method
+    #
+    def parse
+      return unless ::Kernel.test ?f, @fn
+      @section_name = nil
+      ::File.open(@fn, 'r') do |f|
+        while line = f.gets
+          @line_number += 1
+          parse_line line.chomp
+        end
+      end
+      @section_name = nil
+      @line_number  = 0
+      return
+    end
+  
+    #
+    # call-seq:
+    #    set_vall( key, value) => value
+    #
+    # Sets the value of the given key taking the current stack level into account
+    #
+    def set_value key, value
+      begin
+        p = @ini[@section_name]
+        @dict_stack.map { |d| p = (p[d]||={}) }
+        p[key] = value
+      rescue NoMethodError
+        raise ConfigParseError, "sectionless parameter declaration encountered at line #{@line_number}"
+      end
+    end
+    
+    private
+    
+    #
+    # call-seq:
+    #    current_state (param = nil) => state
+    #
+    # Used for outputing the current parameter hash heirarchy in debug mode
+    #
+    def current_state param = nil
+      state = "@ini[:#{@section_name}]"
+      state << @dict_stack.collect { |c| "[:#{c}]" }.join unless @dict_stack.empty?
+      state << "[:#{@in_list_name}]" unless @in_list_name.nil?
+      state << "[:#{param}]" unless param.nil?
+      state
+    end
+  
+    #
+    # call-seq:
+    #    parse_line(line)
+    #
+    # Parses the given line
+    #
+    def parse_line line
+      line.gsub!(/\s+#.*$/, '') # strip comments
+  
+      # replace __FILE__ with the file being parsed
+      line.gsub!('__FILE__', File.expand_path(@fn))
+      
+      # replace __DIR__ with the path of the file being parsed
+      line.gsub!('__DIR__', File.dirname(File.expand_path(@fn)))  
+
+      # replace __ID_<METHOD>__ with Emissary.identity.<method>
+      [ :name, :instance_id, :server_id, :cluster_id, :account_id ].each do |id_method|
+        line.gsub!("__ID_#{id_method.to_s.upcase}__", Emissary.identity.__send__(id_method).to_s)
+      end
+      
+      if not @in_list_name.nil? and line !~ @rgxp_list_stop
+        line = line.strip.split(/\s*,\s*/).compact.reject(&:blank?)
+        Emissary.logger.debug  "  ---> LIST ITEM #{current_state} << #{line.inspect}" if @debug
+        # then we're in the middle of a list item, so add to it
+        @list_items = @list_items | line
+        return
+      end
+      
+      case line
+        # ignore blank lines and comment lines
+        when @rgxp_comment: return
+    
+        # this is a section declaration
+        when @rgxp_section
+          Emissary.logger.debug  "SECTION: #{line}" if @debug
+  
+          unless @in_dict_name.nil?
+            raise ConfigParseError, "dictionary '#{@in_dict_name}' crosses section '#{$1.strip.downcase}' boundary at line #{@line_number}"
+          end
+          
+          @section_name = $1.strip.downcase.to_sym
+          @ini[@section_name] ||= {}
+          
+        when @rgxp_dict_start
+          @dict_stack << $1.strip.downcase.to_sym
+          Emissary.logger.debug  "  ---> DICT_BEG: #{@dict_stack.last}" if @debug
+          
+        when @rgxp_dict_stop
+          raise ConfigParseError, "end of dictionary found without beginning at line #{@line_number}" if @dict_stack.empty?
+          Emissary.logger.debug  "  ---> DICT_END: #{@dict_stack.last}" if @debug
+          @dict_stack.pop 
+          return
+  
+        when @rgxp_list_line
+          list_name = $1.strip.downcase.to_sym
+          list_items = $2.strip.split(/\s*,\s*/).compact.reject(&:blank?)
+          
+          unless not @debug
+            Emissary.logger.debug "  ---> LIST_BEG: #{list_name}"
+            list_items.each do |li|
+              Emissary.logger.debug "  ---> LIST_ITEM: #{current_state list_name} << [\"#{li}\"]"
+            end
+            Emissary.logger.debug "  ---> LIST_END: #{list_name}" 
+          end
+          
+          set_value list_name, list_items
+          
+        when @rgxp_list_start
+          Emissary.logger.debug "  ---> LIST_BEG: #{line}" if @debug
+          @in_list_name = $1.strip.downcase.to_sym
+  
+        when @rgxp_list_stop
+          Emissary.logger.debug "  ---> LIST_END: #{@in_list_name} - #{@list_items.inspect}" if @debug
+          raise ConfigParseError, "end of list found without beginning at line #{@line_number}" if @in_list_name.nil?
+          set_value @in_list_name, @list_items
+          
+          @in_list_name = nil
+          @list_items = []
+          
+        when @rgxp_param
+          val = $2.strip
+          val = val[1..-2] if val[0..0] == "'" || val[-1..-1] == '"'
+    
+          key = $1.strip.downcase.to_sym
+          Emissary.logger.debug  "  ---> PARAM: #{current_state key} = #{val}" if @debug
+          set_value key, val
+  
+      else
+        raise Exception, "Unable to parse line #{@line_number}: #{line}"
+      end
+      return true
+    end
+  end
+end