rouster 0.57 → 0.61

data/lib/rouster.rb

@@ -12,7 +12,7 @@ require 'rouster/vagrant'
 class Rouster
 
   # sporadically updated version number
-  VERSION = 0.57
+  VERSION = 0.61
 
   # custom exceptions -- what else do we want them to include/do?
   class ArgumentError        < StandardError; end # thrown by methods that take parameters from users
@@ -76,15 +76,6 @@ class Rouster
       @verbosity_logfile = 2 # this is kind of arbitrary, but won't actually be created unless opts[:logfile] is also passed
     end
 
-    if opts.has_key?(:sudo)
-      @sudo = opts[:sudo]
-    elsif @passthrough.class.eql?(Hash)
-      # TODO say something here.. or maybe check to see if our user has passwordless sudo?
-      @sudo = false
-    else
-      @sudo = true
-    end
-
     @ostype = nil
     @output = Array.new
     @cache  = Hash.new
@@ -109,26 +100,98 @@ class Rouster
 
     @logger.outputters[0].level = @verbosity_console # can't set this when instantiating a .std* logger, and want the FileOutputter at a different level
 
+    if opts.has_key?(:sudo)
+      @sudo = opts[:sudo]
+    elsif @passthrough.class.eql?(Hash)
+      @logger.debug(sprintf('passthrough without sudo specification, defaulting to false'))
+      @sudo = false
+    else
+      @sudo = true
+    end
+
     if @passthrough
-      # TODO do better about informing of required specifications, maybe point them to an URL?
       @vagrantbinary = 'vagrant' # hacky fix to is_vagrant_running?() grepping, doesn't need to actually be in $PATH
+      @sshtunnel     = opts[:sshtunnel].nil? ? false : @sshtunnel # unless user has specified it, non-local passthroughs default to not open tunnel
+
+      defaults = {
+        :paranoid          => false, # valid overrides are: false, true, :very, or :secure
+        :ssh_sleep_ceiling => 9,
+        :ssh_sleep_time    => 10,
+      }
+
+      @passthrough = defaults.merge(@passthrough)
+
       if @passthrough.class != Hash
         raise ArgumentError.new('passthrough specification should be hash')
       elsif @passthrough[:type].nil?
-        raise ArgumentError.new('passthrough :type must be specified, :local or :remote allowed')
+        raise ArgumentError.new('passthrough :type must be specified, :local, :remote or :aws allowed')
       elsif @passthrough[:type].eql?(:local)
-        @sshtunnel = false
         @logger.debug('instantiating a local passthrough worker')
+        @sshtunnel = opts[:sshtunnel].nil? ? true : opts[:sshtunnel] # override default, if local, open immediately
+
       elsif @passthrough[:type].eql?(:remote)
-        raise ArgumentError.new('remote passthrough requires :host specification') if @passthrough[:host].nil?
-        raise ArgumentError.new('remote passthrough requires :user specification') if @passthrough[:user].nil?
-        raise ArgumentError.new('remote passthrough requires :key specification')  if @passthrough[:key].nil?
+        @logger.debug('instantiating a remote passthrough worker')
+
+        [:host, :user, :key].each do |r|
+          raise ArgumentError.new(sprintf('remote passthrough requires[%s] specification', r)) if @passthrough[r].nil?
+        end
+
         raise ArgumentError.new('remote passthrough requires valid :key specification, should be path to private half') unless File.file?(@passthrough[:key])
         @sshkey = @passthrough[:key] # TODO refactor so that you don't have to do this..
-        @logger.debug('instantiating a remote passthrough worker')
+
+      elsif @passthrough[:type].eql?(:aws) or @passthrough[:type].eql?(:raiden)
+        @logger.debug(sprintf('instantiating an %s passthrough worker', @passthrough[:type]))
+
+        aws_defaults = {
+          :ami                   => 'ami-7bdaa84b', # RHEL 6.5 x64 in us-west-2
+          :dns_propagation_sleep => 30, # how much time to wait after ELB creation before attempting to connect
+          :key_id                => ENV['AWS_ACCESS_KEY_ID'],
+          :min_count             => 1,
+          :max_count             => 1,
+          :region                => 'us-west-2',
+          :secret_key            => ENV['AWS_SECRET_ACCESS_KEY'],
+          :size                  => 't1.micro',
+          :ssh_port              => 22,
+          :user                  => 'ec2-user',
+        }
+
+        if @passthrough.has_key?(:ami)
+          @logger.debug(':ami specified, will start new EC2 instance')
+
+          @passthrough[:security_groups] = @passthrough[:security_groups].is_a?(Array) ? @passthrough[:security_groups] : [ @passthrough[:security_groups] ]
+
+          @passthrough = aws_defaults.merge(@passthrough)
+
+          [:ami, :size, :user, :region, :key, :keypair, :key_id, :secret_key, :security_groups].each do |r|
+            raise ArgumentError.new(sprintf('AWS passthrough requires %s specification', r)) if @passthrough[r].nil?
+          end
+
+        elsif @passthrough.has_key?(:instance)
+          @logger.debug(':instance specified, will connect to existing EC2 instance')
+
+          @passthrough = aws_defaults.merge(@passthrough)
+
+          if @passthrough[:type].eql?(:aws)
+            @passthrough[:host] = self.aws_describe_instance(@passthrough[:instance])['dnsName']
+          else
+            @passthrough[:host] = self.find_ssh_elb(true)
+          end
+
+          [:instance, :key, :user, :host].each do |r|
+            raise ArgumentError.new(sprintf('AWS passthrough requires [%s] specification', r)) if @passthrough[r].nil?
+          end
+
+        else
+          raise ArgumentError.new('AWS passthrough requires either :ami or :instance specification')
+        end
+
+        raise ArgumentError.new('AWS passthrough requires valid :sshkey specification, should be path to private half') unless File.file?(@passthrough[:key])
+        @sshkey = @passthrough[:key]
+
       else
-        raise ArgumentError.new(sprintf('passthrough :type [%s] unknown, allowed: :local, :remote', @passthrough[:type]))
+        raise ArgumentError.new(sprintf('passthrough :type [%s] unknown, allowed: :aws, :local, :remote', @passthrough[:type]))
       end
+
     else
 
       @logger.debug('Vagrantfile and VM name validation..')
@@ -154,14 +217,6 @@ class Rouster
       end
     end
 
-    # this is breaking test/functional/test_caching.rb test_ssh_caching (if the VM was not running when the test started)
-    # it slows down object instantiation, but is a good test to ensure the machine name is valid..
-    begin
-      self.status()
-    rescue Rouster::LocalExecutionError => e
-      raise InternalError.new(sprintf('caught non-0 exitcode from status(): %s', e.message))
-    end
-
     begin
       raise InternalError.new('ssh key not specified') if @sshkey.nil?
       raise InternalError.new('ssh key does not exist') unless File.file?(@sshkey)
@@ -282,23 +337,22 @@ class Rouster
 
     if @ssh.nil? or @ssh.closed?
       begin
-        self.connect_ssh_tunnel()
+        res = self.connect_ssh_tunnel()
       rescue Rouster::InternalError, Net::SSH::Disconnect => e
         res = false
       end
 
     end
 
-    if res.nil?
+    if res.nil? or res.is_a?(Net::SSH::Connection::Session)
       begin
         self.run('echo functional test of SSH tunnel')
+          res = true
       rescue
         res = false
       end
     end
 
-    res = true if res.nil?
-
     if @cache_timeout
       @cache[:is_available_via_ssh?] = Hash.new unless @cache[:is_available_via_ssh?].class.eql?(Hash)
       @cache[:is_available_via_ssh?][:time] = Time.now.to_i
@@ -360,18 +414,36 @@ class Rouster
   def connect_ssh_tunnel
 
     if self.is_passthrough?
-      if self.passthrough[:type].eql?(:local)
+      if @passthrough[:type].eql?(:local)
+        @logger.debug("local passthroughs don't need ssh tunnel, shell execs are used")
+        return false
+      elsif @passthrough[:host].nil?
+        @logger.info(sprintf('not attempting to connect, no known hostname for[%s]', self.passthrough))
         return false
       else
-        @logger.debug('opening remote SSH tunnel..')
-        @ssh = Net::SSH.start(
-          @passthrough[:host],
-          @passthrough[:user],
-          :port => @passthrough[:port],
-          :keys => [ @passthrough[:key] ],
-          :paranoid => false
-        )
+        ceiling    = @passthrough[:ssh_sleep_ceiling]
+        sleep_time = @passthrough[:ssh_sleep_time]
+
+        0.upto(ceiling) do |try|
+          @logger.debug(sprintf('opening remote SSH tunnel[%s]..', @passthrough[:host]))
+          begin
+            @ssh = Net::SSH.start(
+              @passthrough[:host],
+              @passthrough[:user],
+              :port => @passthrough[:ssh_port],
+              :keys => [ @passthrough[:key] ], # TODO this should be @sshkey
+              :paranoid => false
+            )
+            break
+          rescue => e
+            raise e if try.eql?(ceiling) # eventually want to throw a SocketError
+            @logger.debug(sprintf('failed to open tunnel[%s], trying again in %ss', e.message, sleep_time))
+            sleep sleep_time
+          end
+        end
       end
+      @logger.debug(sprintf('successfully opened SSH tunnel to[%s]', passthrough[:host]))
+
     else
       # not a passthrough, normal connection
       status = self.status()
@@ -387,6 +459,7 @@ class Rouster
           :paranoid => false
         )
       else
+        # TODO will we ever hit this? or will we be thrown first?
         raise InternalError.new(sprintf('VM is not running[%s], unable open SSH tunnel', status))
       end
     end
@@ -416,32 +489,25 @@ class Rouster
       return @ostype
     end
 
-    # TODO switch to file based detection
-    # Ubuntu - /etc/os-release
-    # Solaris - /etc/release
-    # RHEL/CentOS - /etc/redhat-release
-    # OSX - ?
+    files = {
+      :ubuntu  => '/etc/os-release', # debian too
+      :solaris => '/etc/release',
+      :redhat  => '/etc/redhat-release', # centos too
+      :osx     => '/System/Library/CoreServices/SystemVersion.plist',
+    }
 
     res   = nil
-    uname = self.run('uname -a')
-
-    case uname
-      when /Darwin/i
-        res = :osx
-      when /SunOS|Solaris/i
-        res =:solaris
-      when /Ubuntu/i
-        res = :ubuntu
-      when /Debian/i
-        res = :debian
-      else
-        if self.is_file?('/etc/redhat-release')
-          res = :redhat
-        else
-          res = nil
-        end
+
+    files.each do |os,file|
+      if self.is_file?(file)
+        @logger.debug(sprintf('determined OS to be[%s] via[%s]', os, file))
+        res = os
+        break
+      end
     end
 
+    @logger.error(sprintf('unable to determine OS, looking for[%s]', files)) if res.nil?
+
     @ostype = res
     res
   end
@@ -493,6 +559,7 @@ class Rouster
       raise FileTransferError.new(sprintf('unable to put[%s], exception[%s]', local_file, e.message()))
     end
 
+    return true
   end
 
   ##

data/lib/rouster/deltas.rb

@@ -213,9 +213,9 @@ class Rouster
   #
   # supported OS
   # * OSX - runs `pkgutil --pkgs` and `pkgutil --pkg-info=<package>` (if deep)
-  # * RedHat - runs `rpm -qa`
+  # * RedHat - runs `rpm -qa --qf "%{n}@%{v}@%{arch}\n"` (does not support deep)
   # * Solaris - runs `pkginfo` and `pkginfo -l <package>` (if deep)
-  # * Ubuntu - runs `dpkg --get-selections` and `dpkg -s <package>` (if deep)
+  # * Ubuntu - runs `dpkg-query -W -f='${Package}\@${Version}\@${Architecture}\n'` (does not support deep)
   #
   # raises InternalError if unsupported operating system
   def get_packages(cache=true, deep=true)
@@ -239,62 +239,93 @@ class Rouster
 
       raw = self.run('pkgutil --pkgs')
       raw.split("\n").each do |line|
+        name    = line
+        arch    = '?'
         version = '?'
 
         if deep
           # can get install time, volume and location as well
-          local_res = self.run(sprintf('pkgutil --pkg-info=%s', line))
-          version = $1 if local_res.match(/version\:\s+(.*?)$/)
+          local_res = self.run(sprintf('pkgutil --pkg-info=%s', name))
+          version   = $1 if local_res.match(/version\:\s+(.*?)$/)
+        end
+
+        if res.has_key?(name)
+          # different architecture of an already known package
+          @logger.debug(sprintf('found package with already known name[%s], value[%s], new line[%s], turning into array', name, res[name], line))
+          new_element = { :version => version, :arch => arch }
+          res[name]   = [ res[name], new_element ]
+        else
+          res[name] = { :version => version, :arch => arch }
         end
 
-        res[line] = version
       end
 
     elsif os.eql?(:solaris)
       raw = self.run('pkginfo')
       raw.split("\n").each do |line|
-        next if line.match(/(.*?)\s+(.*?)\s(.*)$/).empty?
+        next if line.match(/(.*?)\s+(.*?)\s(.*)$/).nil?
         name    = $2
+        arch    = '?'
         version = '?'
 
         if deep
-          local_res = self.run(sprintf('pkginfo -l %s', name))
-          version   = $1 if local_res.match(/VERSION\:\s+(.*?)$/i)
+          begin
+            # who throws non-0 exit codes when querying for legit package information? solaris does.
+            local_res = self.run(sprintf('pkginfo -l %s', name))
+            arch      = $1 if local_res.match(/ARCH\:\s+(.*?)$/)
+            version   = $1 if local_res.match(/VERSION\:\s+(.*?)$/)
+          rescue
+            arch    = '?' if arch.nil?
+            version = '?' if arch.nil?
+          end
         end
 
-        res[name] = version
+        if res.has_key?(name)
+          # different architecture of an already known package
+          @logger.debug(sprintf('found package with already known name[%s], value[%s], new line[%s], turning into array', name, res[name], line))
+          new_element = { :version => version, :arch => arch }
+          res[name]   = [ res[name], new_element ]
+        else
+          res[name] = { :version => version, :arch => arch }
+        end
       end
 
     elsif os.eql?(:ubuntu) or os.eql?(:debian)
-      raw = self.run('dpkg --get-selections')
+      raw = self.run("dpkg-query -W -f='${Package}@${Version}@${Architecture}\n'")
       raw.split("\n").each do |line|
-        next if line.match(/^(.*?)\s/).nil?
+        next if line.match(/(.*?)\@(.*?)\@(.*)/).nil?
         name    = $1
-        version = '?'
-
-        if deep
-          local_res = self.run(sprintf('dpkg -s %s', name))
-          version   = $1 if local_res.match(/Version\:\s(.*?)$/)
+        version = $2
+        arch    = $3
+
+        if res.has_key?(name)
+          # different architecture of an already known package
+          @logger.debug(sprintf('found package with already known name[%s], value[%s], new line[%s], turning into array', name, res[name], line))
+          new_element = { :version => version, :arch => arch }
+          res[name]   = [ res[name], new_element ]
+        else
+          res[name] = { :version => version, :arch => arch }
         end
 
-        res[name] = version
       end
 
     elsif os.eql?(:redhat)
-      raw = self.run('rpm -qa')
+      raw = self.run('rpm -qa --qf "%{n}@%{v}@%{arch}\n"')
       raw.split("\n").each do |line|
-        next if line.match(/(.*?)-(\d*\..*)/).nil? # ht petersen.allen
-        #next if line.match(/(.*)-(\d+\.\d+.*)/).nil? # another alternate, but still not perfect
+        next if line.match(/(.*?)\@(.*?)\@(.*)/).nil?
         name    = $1
-        version = '?' # we could use $2, but we don't trust it
-
-        if deep
-          local_res = self.run(sprintf('rpm -qi %s', line))
-          name    = $1 if local_res.match(/Name\s+:\s(\S*)/)
-          version = $1 if local_res.match(/Version\s+:\s(\S*)/)
+        version = $2
+        arch    = $3
+
+        if res.has_key?(name)
+          # different architecture of an already known package
+          @logger.debug(sprintf('found package with already known name[%s], value[%s], new line[%s], turning into array', name, res[name], line))
+          new_element = { :version => version, :arch => arch }
+          res[name]   = [ res[name], new_element ]
+        else
+          res[name] = { :version => version, :arch => arch }
         end
 
-        res[name] = version
       end
 
     else
@@ -389,17 +420,20 @@ class Rouster
   # parameters
   # * [cache]    - boolean controlling whether data retrieved/parsed is cached, defaults to true
   # * [humanize] - boolean controlling whether data retrieved is massaged into simplified names or returned mostly as retrieved
+  # * [type]     - symbol indicating which service controller to query, defaults to :all
+  # * [seed]     - test hook to seed the output of service commands
   #
-  # supported OS
-  # * OSX - runs `launchctl list`
-  # * RedHat - runs `/sbin/service --status-all`
-  # * Solaris - runs `svcs`
-  # * Ubuntu - runs `service --status-all`
+  # supported OS and types
+  # * OSX     - :launchd
+  # * RedHat  - :systemv or :upstart
+  # * Solaris - :smf
+  # * Ubuntu  - :systemv or :upstart
   #
   # notes
   # * raises InternalError if unsupported operating system
   # * OSX, Solaris and Ubuntu/Debian will only return running|stopped|unsure, the exists|installed|operational modes are RHEL/CentOS only
-  def get_services(cache=true, humanize=true)
+
+  def get_services(cache=true, humanize=true, type=:all, seed=nil)
     if cache and ! self.deltas[:services].nil?
 
       if self.cache_timeout and self.cache_timeout.is_a?(Integer) and (Time.now.to_i - self.cache[:services]) > self.cache_timeout
@@ -415,132 +449,234 @@ class Rouster
     res = Hash.new()
     os  = self.os_type
 
-    allowed_modes = %w(exists installed operational running stopped unsure)
-    failover_mode = 'unsure'
+    commands = {
+      :osx => {
+        :launchd => 'launchctl list',
+      },
+      :solaris => {
+        :smf => 'svcs -a',
+      },
+
+      # TODO we really need to implement something like osfamily
+      :ubuntu => {
+        :systemv => 'service --status-all 2>&1',
+        :upstart => 'initctl list',
+      },
+      :debian => {
+        :systemv => 'service --status-all 2>&1',
+        :upstart => 'initctl list',
+      },
+      :redhat => {
+        :systemv => '/sbin/service --status-all',
+        :upstart => 'initctl list',
+      },
+    }
+
+    if type.eql?(:all)
+      type = commands[os].keys
+    end
 
-    if os.eql?(:osx)
+    type = type.class.eql?(Array) ? type : [ type ]
 
-      raw = self.run('launchctl list')
-      raw.split("\n").each do |line|
-        next if line.match(/(?:\S*?)\s+(\S*?)\s+(\S*)$/).nil?
+    type.each do |provider|
 
-        service = $2
-        mode    = $1
+      raise InternalError.new(sprintf('unable to get service information from VM operating system[%s]', os)) unless commands.has_key?(os)
+      raise ArgumentError.new(sprintf('unable to find command provider[%s] for [%s]', provider, os))  if commands[os][provider].nil?
 
-        if humanize # should we do this with a .freeze instead?
-          if mode.match(/^\d/)
-            mode = 'running'
-          else
-            mode = 'stopped'
-          end
-        end
+      @logger.info(sprintf('get_services using provider [%s] on [%s]', provider, os))
 
-        res[service] = mode
-      end
+      # TODO while this is true, what if self.user is 'root'.. -- the problem is we don't have self.user, and we store this data differently depending on self.passthrough?
+      @logger.warn('gathering service information typically works better with sudo, which is currently not being used') unless self.uses_sudo?
 
-    elsif os.eql?(:solaris)
+      # TODO come up with a better test hook -- real problem is that we can't seed 'raw' with different values per iteration
+      raw = seed.nil? ? self.run(commands[os][provider]) : seed
 
-      raw = self.run('svcs -a')
-      raw.split("\n").each do |line|
-        next if line.match(/(.*?)\s+(?:.*?)\s+(.*?)$/).nil?
-
-        service = $2
-        mode    = $1
-
-        if humanize
-          if mode.match(/^online/)
-            mode = 'running'
-          elsif mode.match(/^legacy_run/)
-            mode = 'running'
-          elsif mode.match(/^disabled/)
-            mode = 'stopped'
-          end
+      if os.eql?(:osx)
+
+        raw.split("\n").each do |line|
+          next if line.match(/(?:\S*?)\s+(\S*?)\s+(\S+)$/).nil?
+          tokens = line.split("\s")
+          service = tokens[-1]
+          mode    = tokens[0]
 
-          if service.match(/^svc:\/.*\/(.*?):.*/)
-            # turning 'svc:/network/cswpuppetd:default' into 'cswpuppetd'
-            service = $1
-          elsif service.match(/^lrc:\/.*?\/.*\/(.*)/)
-            # turning 'lrc:/etc/rcS_d/S50sk98Sol' into 'S50sk98Sol'
-            service = $1
+          if humanize # should we do this with a .freeze instead?
+            if mode.match(/^\d/)
+              mode = 'running'
+            elsif mode.match(/-/)
+              mode = 'stopped'
+            else
+              next # this should handle the banner "PID     Status  Label"
+            end
           end
+
+          res[service] = mode
         end
 
-        res[service] = mode
+      elsif os.eql?(:solaris)
 
-      end
+        raw.split("\n").each do |line|
+          next if line.match(/(.*?)\s+(?:.*?)\s+(.*?)$/).nil?
 
-    elsif os.eql?(:ubuntu) or os.eql?(:debian)
+          service = $2
+          mode    = $1
 
-      raw = self.run('service --status-all 2>&1')
-      raw.split("\n").each do |line|
-        next if line.match(/\[(.*?)\]\s+(.*)$/).nil?
-        mode    = $1
-        service = $2
-
-        if humanize
-          mode = 'stopped' if mode.match('-')
-          mode = 'running' if mode.match('\+')
-          mode = 'unsure'  if mode.match('\?')
-        end
+          if humanize
+            if mode.match(/^online/)
+              mode = 'running'
+            elsif mode.match(/^legacy_run/)
+              mode = 'running'
+            elsif mode.match(/^disabled/)
+              mode = 'stopped'
+            end
 
-        res[service] = mode
-      end
+            if service.match(/^svc:\/.*\/(.*?):.*/)
+              # turning 'svc:/network/cswpuppetd:default' into 'cswpuppetd'
+              service = $1
+            elsif service.match(/^lrc:\/.*?\/.*\/(.*)/)
+              # turning 'lrc:/etc/rcS_d/S50sk98Sol' into 'S50sk98Sol'
+              service = $1
+            end
+          end
 
-    elsif os.eql?(:redhat)
+          res[service] = mode
 
-      raw = self.run('/sbin/service --status-all')
-      raw.split("\n").each do |line|
+        end
 
-        if humanize
+      elsif os.eql?(:ubuntu) or os.eql?(:debian)
+
+        raw.split("\n").each do |line|
+          if provider.eql?(:systemv)
+              next if line.match(/\[(.*?)\]\s+(.*)$/).nil?
+              mode    = $1
+              service = $2
+
+              if humanize
+                mode = 'stopped' if mode.match('-')
+                mode = 'running' if mode.match('\+')
+                mode = 'unsure'  if mode.match('\?')
+              end
+
+              res[service] = mode
+          elsif provider.eql?(:upstart)
+              if line.match(/(.*?)\s.*?(.*?),/)
+                # tty (/dev/tty3) start/running, process 1601
+                # named start/running, process 8959
+                service = $1
+                mode    = $2
+              elsif line.match(/(.*?)\s(.*)/)
+                # rcS stop/waiting
+                service = $1
+                mode    = $2
+              else
+                @logger.warn("unable to process upstart line[#{line}], skipping")
+                next
+              end
+
+              if humanize
+                mode = 'stopped' if mode.match('stop/waiting')
+                mode = 'running' if mode.match('start/running')
+                mode = 'unsure'  unless mode.eql?('stopped') or mode.eql?('running')
+              end
+
+              res[service] = mode
+          end
+        end
 
-          if line.match(/^(\w+?)\sis\s(.*)$/)
-            # <service> is <state>
-            name = $1
-            state = $2
-            res[name] = state
+      elsif os.eql?(:redhat)
+
+        raw.split("\n").each do |line|
+          if provider.eql?(:systemv)
+            if humanize
+              if line.match(/^(\w+?)\sis\s(.*)$/)
+                # <service> is <state>
+                name = $1
+                state = $2
+                res[name] = state
+
+                if state.match(/^not/)
+                  # this catches 'Kdump is not operational'
+                  res[name] = 'stopped'
+                end
+
+              elsif line.match(/^(\w+?)\s\(pid.*?\)\sis\s(\w+)$/)
+                # <service> (pid <pid> [pid]) is <state>...
+                res[$1] = $2
+              elsif line.match(/^(\w+?)\sis\s(\w+)\.*$/) # not sure this is actually needed
+                @logger.debug('triggered supposedly unnecessary regex')
+                # <service> is <state>. whatever
+                res[$1] = $2
+              elsif line.match(/razor_daemon:\s(\w+).*$/)
+                # razor_daemon: running [pid 11325]
+                # razor_daemon: no instances running
+                res['razor_daemon'] = $1.eql?('running') ? $1 : 'stopped'
+              elsif line.match(/^(\w+?)\:.*?(\w+)$/)
+                # <service>: whatever <state>
+                res[$1] = $2
+              elsif line.match(/^(\w+?):.*?\sis\snot\srunning\.$/)
+                # ip6tables: Firewall is not running.
+                res[$1] = 'stopped'
+              elsif line.match(/^(\w+?)\s.*?\s(.*)$/)
+                # netconsole module not loaded
+                state = $2
+                res[$1] = $2.match(/not/) ? 'stopped' : 'running'
+              elsif line.match(/^(\w+)\s(\w+).*$/)
+                # <process> <state> whatever
+                res[$1] = $2
+              else
+                # original regex implementation, if we didn't match anything else, failover to this
+                next if line.match(/^([^\s:]*).*\s(\w*)(?:\.?){3}$/).nil?
+                res[$1] = $2
+              end
+            else
+              next if line.match(/^([^\s:]*).*\s(\w*)(?:\.?){3}$/).nil?
+              res[$1] = $2
+            end
+          elsif provider.eql?(:upstart)
+
+            if line.match(/(.*?)\s.*?(.*?),/)
+              # tty (/dev/tty3) start/running, process 1601
+              # named start/running, process 8959
+              service = $1
+              mode    = $2
+            elsif line.match(/(.*?)\s(.*)/)
+              # rcS stop/waiting
+              service = $1
+              mode    = $2
+            else
+              @logger.warn("unable to process upstart line[#{line}], skipping")
+              next
+            end
 
-            if state.match(/^not/)
-              # this catches 'Kdump is not operational'
-              res[name] = 'stopped'
+            if humanize
+              mode = 'stopped' if mode.match('stop/waiting')
+              mode = 'running' if mode.match('start/running')
+              mode = 'unsure'  unless mode.eql?('stopped') or mode.eql?('running')
             end
 
-          elsif line.match(/^(\w+?)\s\(pid.*?\)\sis\s(\w+)$/)
-            # <service> (pid <pid> [pid]) is <state>...
-            res[$1] = $2
-          elsif line.match(/^(\w+?)\sis\s(\w+)\.*$/) # not sure this is actually needed
-            @logger.debug('triggered supposedly unnecessary regex')
-            # <service> is <state>. whatever
-            res[$1] = $2
-          elsif line.match(/^(\w+?)\:.*?(\w+)$/)
-            # <service>: whatever <state>
-            res[$1] = $2
-          elsif line.match(/^(\w+)\s(\w+).*$/)
-            # <process> <state> whatever
-            res[$1] = $2
-          else
-            # original regex implementation, if we didn't match anything else, failover to this
-            next if line.match(/^([^\s:]*).*\s(\w*)(?:\.?){3}$/).nil?
-            res[$1] = $2
+            res[service] = mode unless res.has_key?(service)
+
+
           end
 
-        else
-          next if line.match(/^([^\s:]*).*\s(\w*)(?:\.?){3}$/).nil?
-          res[$1] = $2
         end
 
+        # end of os casing
       end
 
-      # issue #63 handling
-      if humanize
-        res.each_pair do |k,v|
-          next if allowed_modes.member?(v)
-          @logger.debug(sprintf('replacing service[%s] status of [%s] with [%s] for uniformity', k, v, failover_mode))
-          res[k] = failover_mode
-        end
-      end
+      # end of provider processing
+    end
 
-    else
-      raise InternalError.new(sprintf('unable to get service information from VM operating system[%s]', os))
+    # issue #63 handling
+    # TODO should we consider using symbols here instead?
+    allowed_modes = %w(exists installed operational running stopped unsure)
+    failover_mode = 'unsure'
+
+    if humanize
+      res.each_pair do |k,v|
+        next if allowed_modes.member?(v)
+        @logger.debug(sprintf('replacing service[%s] status of [%s] with [%s] for uniformity', k, v, failover_mode))
+        res[k] = failover_mode
+      end
     end
 
     if cache