oxidized 0.29.1 → 0.37.0

data/lib/oxidized/input/http.rb

@@ -1,18 +1,15 @@
 module Oxidized
-  require "oxidized/input/cli"
   require "net/http"
   require "json"
+  require "net/http/digest_auth"
 
   class HTTP < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
       @secure = false
       @username = nil
       @password = nil
       @headers = {}
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-http", "w") if Oxidized.config.input.debug?
       @node.model.cfg["http"].each { |cb| instance_exec(&cb) }
 
       return true unless @main_page && defined?(login)
@@ -47,26 +44,79 @@ module Oxidized
     private
 
     def get_http(path)
-      schema = @secure ? "https://" : "http://"
-      uri = URI("#{schema}#{@node.ip}#{path}")
-      req = Net::HTTP::Get.new(uri)
-      req.basic_auth @username, @password unless @username.nil?
-      @headers.each do |header, value|
-        req.add_field(header, value)
-      end
+      res = perform_http_request(path, method: :get)
+      res.body
+    end
+
+    def post_http(path, body = nil, extra_headers = {})
+      res = perform_http_request(path, method: :post, body: body, extra_headers: extra_headers)
+      res.body
+    end
+
+    def perform_http_request(path, method: :get, body: nil, extra_headers: {})
+      uri = get_uri(path)
+      http_method = method.to_s.upcase
+
+      logger.debug "Making #{http_method} request to: #{uri}"
+
       ssl_verify = Oxidized.config.input.http.ssl_verify? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
-      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https", verify_mode: ssl_verify) do |http|
+
+      res = make_request(uri, ssl_verify, extra_headers, method: method, body: body)
+
+      if res.code == '401' && res['www-authenticate']&.include?('Digest')
+        uri.user = @username
+        uri.password = URI.encode_www_form_component(@password)
+        logger.debug "Server requires Digest authentication"
+
+        auth = Net::HTTP::DigestAuth.new.auth_header(uri, res['www-authenticate'], http_method)
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => auth),
+                           method: method, body: body)
+
+      elsif @username && @password && !authorization_header_present?(extra_headers)
+        logger.debug "Falling back to Basic authentication"
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => basic_auth_header),
+                           method: method, body: body)
+      end
+
+      logger.debug "Response code: #{res.code}"
+      res
+    end
+
+    def make_request(uri, ssl_verify, extra_headers = {}, method: :get, body: nil)
+      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https", verify_mode: ssl_verify) do |http|
+        req_class = if method == :get
+                      Net::HTTP::Get
+                    elsif method == :post
+                      Net::HTTP::Post
+                    else
+                      raise Oxidized::OxidizedError, "Unsupported HTTP method: #{method.inspect}. " \
+                                                     "Only :get and :post are supported"
+                    end
+        req = req_class.new(uri)
+        @headers.merge(extra_headers).each { |header, value| req.add_field(header, value) }
+        req.body = body if body
+
+        logger.debug "Sending #{method.to_s.upcase} request with headers: #{@headers.merge(extra_headers)}"
         http.request(req)
       end
-      res.body
     end
 
-    def log(str)
-      @log&.write(str)
+    def authorization_header_present?(headers)
+      headers.keys.any? { |key| key.to_s.downcase == 'authorization' }
     end
 
-    def disconnect
-      @log.close if Oxidized.config.input.debug?
+    def basic_auth_header
+      "Basic " + ["#{@username}:#{@password}"].pack('m').delete("\r\n")
+    end
+
+    def disconnect; end
+
+    def get_uri(path)
+      path = URI.parse(path)
+      uri_class = @secure ? URI::HTTPS : URI::HTTP
+      uri_class.build(host:  @node.ip,
+                      path:  path.path,
+                      query: path.query)
     end
   end
 end

data/lib/oxidized/input/input.rb

@@ -1,23 +1,44 @@
+require_relative 'cli'
+
 module Oxidized
   class PromptUndetect < OxidizedError; end
 
   class Input
+    include SemanticLogger::Loggable
     include Oxidized::Config::Vars
+    include Oxidized::Input::CLI
 
-    RescueFail = {
-      debug: [
-        Errno::ECONNREFUSED
-      ],
-      warn:  [
-        IOError,
-        PromptUndetect,
-        Timeout::Error,
-        Errno::ECONNRESET,
-        Errno::EHOSTUNREACH,
-        Errno::ENETUNREACH,
-        Errno::EPIPE,
-        Errno::ETIMEDOUT
-      ]
+    RESCUE_FAIL = {
+      Errno::ECONNREFUSED => :debug,
+      IOError             => :warn,
+      PromptUndetect      => :warn,
+      Timeout::Error      => :warn,
+      Errno::ECONNRESET   => :warn,
+      Errno::EHOSTUNREACH => :warn,
+      Errno::ENETUNREACH  => :warn,
+      Errno::EPIPE        => :warn,
+      Errno::ETIMEDOUT    => :warn
     }.freeze
+
+    # Returns a hash mapping exception classes to their log level
+    def self.rescue_fail
+      RESCUE_FAIL.dup
+    end
+
+    def self.config_name
+      name.split('::').last.downcase
+    end
+
+    def self.to_sym
+      config_name.to_sym
+    end
+
+    def config_name
+      self.class.config_name
+    end
+
+    def to_sym
+      self.class.to_sym
+    end
   end
 end

data/lib/oxidized/input/scp.rb

@@ -0,0 +1,27 @@
+module Oxidized
+  require 'net/ssh'
+  begin
+    require 'net/scp'
+  rescue LoadError
+    raise OxidizedError, 'net/scp not found: sudo gem install net-scp'
+  end
+  require 'timeout'
+  require_relative 'sshbase'
+
+  class SCP < SSHBase
+    RESCUE_FAIL = {
+      Net::SCP::Error => :warn
+    }.freeze
+
+    def self.rescue_fail
+      super.merge(RESCUE_FAIL)
+    end
+
+    def cmd(file)
+      logger.debug "SCP: #{file} @ #{@node.name}"
+      Timeout.timeout(@node.timeout) do
+        @ssh.scp.download!(file)
+      end
+    end
+  end
+end

data/lib/oxidized/input/ssh.rb

@@ -1,29 +1,30 @@
+require 'timeout'
+require_relative 'sshbase'
+require_relative 'debugyaml'
+require_relative 'debugtext'
+
 module Oxidized
-  require 'net/ssh'
-  require 'net/ssh/proxy/command'
-  require 'timeout'
-  require 'oxidized/input/cli'
-  class SSH < Input
-    RescueFail = {
-      debug: [
-        Net::SSH::Disconnect
-      ],
-      warn:  [
-        RuntimeError,
-        Net::SSH::AuthenticationFailed
-      ]
-    }.freeze
-    include Input::CLI
+  class SSH < SSHBase
     class NoShell < OxidizedError; end
 
-    def connect(node)
+    RESCUE_FAIL = {
+      RuntimeError => :warn
+    }.freeze
+
+    def self.rescue_fail
+      super.merge(RESCUE_FAIL)
+    end
+
+    def connect(node) # rubocop:disable Naming/PredicateMethod
       @node        = node
-      @output      = ''
+      @output      = String.new('')
       @pty_options = { term: "vt100" }
       @node.model.cfg['ssh'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-ssh", 'w') if Oxidized.config.input.debug?
 
-      Oxidized.logger.debug "lib/oxidized/input/ssh.rb: Connecting to #{@node.name}"
+      @yaml_debug = DebugYAML.new(Oxidized.config.input.debug, @node, config_name)
+      @text_debug = DebugText.new(Oxidized.config.input.debug, @node, config_name)
+
+      logger.debug "Connecting to #{@node.name}"
       @ssh = Net::SSH.start(@node.ip, @node.auth[:username], make_ssh_opts)
       unless @exec
         shell_open @ssh
@@ -36,20 +37,31 @@ module Oxidized
       connected?
     end
 
-    def connected?
-      @ssh && (not @ssh.closed?)
-    end
-
     def cmd(cmd, expect = node.prompt)
-      Oxidized.logger.debug "lib/oxidized/input/ssh.rb #{cmd} @ #{node.name} with expect: #{expect.inspect}"
-      if @exec
-        @ssh.exec! cmd
-      else
-        cmd_shell(cmd, expect).gsub(/\r\n/, "\n")
+      unless cmd.is_a?(String)
+        logger.error "cmd must be a String (#{cmd.class}): #{cmd.inspect} @ #{node.name}"
+        raise ArgumentError, "cmd must be a String"
       end
+      logger.debug "Sending '#{cmd.dump}' @ #{node.name} with expect: #{expect.inspect}"
+      cmd_output = if @exec
+                     @yaml_debug&.send_data(cmd)
+                     @text_debug&.send_data(cmd)
+                     @ssh.exec! cmd
+                   else
+                     cmd_shell(cmd, expect).gsub("\r\n", "\n")
+                   end
+
+      # only logging @exec as cmd_shell is handled in the ssh loop
+      @yaml_debug&.receive_data(cmd_output) if @exec
+      @text_debug&.receive_data(cmd_output) if @exec
+
+      # Make sure we return a String
+      cmd_output.to_s
     end
 
     def send(data)
+      @yaml_debug&.send_data(data)
+      @text_debug&.send_data(data)
       @ses.send_data data
     end
 
@@ -61,23 +73,27 @@ module Oxidized
 
     private
 
+    # We need a specific disconnect for SSH in shell mode, see issue #3725
     def disconnect
       disconnect_cli
       # if disconnect does not disconnect us, give up after timeout
-      Timeout.timeout(Oxidized.config.timeout) { @ssh.loop }
-    rescue Errno::ECONNRESET, Net::SSH::Disconnect, IOError
+      Timeout.timeout(@node.timeout) { @ssh.loop }
+    rescue Errno::ECONNRESET, Net::SSH::Disconnect, IOError => e
+      logger.debug 'The other side closed the connection while ' \
+                   "disconnecting, raising #{e.class} with #{e.message}"
+    rescue Timeout::Error
+      logger.debug "#{@node.name} timed out while disconnecting"
     ensure
-      @log.close if Oxidized.config.input.debug?
-      (@ssh.close rescue true) unless @ssh.closed?
+      @yaml_debug&.close
+      @text_debug&.close
+      (@ssh.close rescue true) unless @ssh.closed? # rubocop:disable Style/RedundantParentheses
     end
 
     def shell_open(ssh)
       @ses = ssh.open_channel do |ch|
         ch.on_data do |_ch, data|
-          if Oxidized.config.input.debug?
-            @log.print data
-            @log.flush
-          end
+          @yaml_debug&.receive_data(data)
+          @text_debug&.receive_data(data)
           @output << data
           @output = @node.model.expects @output
         end
@@ -98,8 +114,11 @@ module Oxidized
     end
 
     def cmd_shell(cmd, expect_re)
-      @output = ''
-      @ses.send_data cmd + "\n"
+      @output = String.new('')
+
+      @yaml_debug&.send_data(cmd + newline)
+      @text_debug&.send_data(cmd + newline)
+      @ses.send_data cmd + newline
       @ses.process
       expect expect_re if expect_re
       @output
@@ -107,8 +126,8 @@ module Oxidized
 
     def expect(*regexps)
       regexps = [regexps].flatten
-      Oxidized.logger.debug "lib/oxidized/input/ssh.rb: expecting #{regexps.inspect} at #{node.name}"
-      Timeout.timeout(Oxidized.config.timeout) do
+      logger.debug "Expecting #{regexps.inspect} at #{node.name}"
+      Timeout.timeout(@node.timeout) do
         @ssh.loop(0.1) do
           sleep 0.1
           match = regexps.find { |regexp| @output.match regexp }
@@ -118,48 +137,5 @@ module Oxidized
         end
       end
     end
-
-    def make_ssh_opts
-      secure = Oxidized.config.input.ssh.secure?
-      ssh_opts = {
-        number_of_password_prompts:      0,
-        keepalive:                       vars(:ssh_no_keepalive) ? false : true,
-        verify_host_key:                 secure ? :always : :never,
-        append_all_supported_algorithms: true,
-        password:                        @node.auth[:password],
-        timeout:                         Oxidized.config.timeout,
-        port:                            (vars(:ssh_port) || 22).to_i,
-        forward_agent:                   false
-      }
-
-      auth_methods = vars(:auth_methods) || %w[none publickey password]
-      ssh_opts[:auth_methods] = auth_methods
-      Oxidized.logger.debug "AUTH METHODS::#{auth_methods}"
-
-      ssh_opts[:proxy] = make_ssh_proxy_command(vars(:ssh_proxy), vars(:ssh_proxy_port), secure) if vars(:ssh_proxy)
-
-      ssh_opts[:keys]       = [vars(:ssh_keys)].flatten           if vars(:ssh_keys)
-      ssh_opts[:kex]        = vars(:ssh_kex).split(/,\s*/)        if vars(:ssh_kex)
-      ssh_opts[:encryption] = vars(:ssh_encryption).split(/,\s*/) if vars(:ssh_encryption)
-      ssh_opts[:host_key]   = vars(:ssh_host_key).split(/,\s*/)   if vars(:ssh_host_key)
-      ssh_opts[:hmac]       = vars(:ssh_hmac).split(/,\s*/)       if vars(:ssh_hmac)
-
-      if Oxidized.config.input.debug?
-        ssh_opts[:logger]  = Oxidized.logger
-        ssh_opts[:verbose] = Logger::DEBUG
-      end
-
-      ssh_opts
-    end
-
-    def make_ssh_proxy_command(proxy_host, proxy_port, secure)
-      return nil unless !proxy_host.nil? && !proxy_host.empty?
-
-      proxy_command =  "ssh "
-      proxy_command += "-o StrictHostKeyChecking=no " unless secure
-      proxy_command += "-p #{proxy_port} "            if proxy_port
-      proxy_command += "#{proxy_host} -W [%h]:%p"
-      Net::SSH::Proxy::Command.new(proxy_command)
-    end
   end
 end

data/lib/oxidized/input/sshbase.rb

@@ -0,0 +1,102 @@
+module Oxidized
+  require 'net/ssh'
+  require 'net/ssh/proxy/command'
+  require 'timeout'
+
+  class SSHBase < Input
+    RESCUE_FAIL = {
+      Net::SSH::Disconnect           => :debug,
+      Net::SSH::ConnectionTimeout    => :debug,
+      Net::SSH::AuthenticationFailed => :warn,
+      Net::SSH::HostKeyUnknown       => :warn
+    }.freeze
+
+    def self.rescue_fail
+      super.merge(RESCUE_FAIL)
+    end
+
+    def connect(node) # rubocop:disable Naming/PredicateMethod
+      @node = node
+      @node.model.cfg[config_name].each { |cb| instance_exec(&cb) }
+      logger.debug "Connecting to #{@node.name}"
+      @ssh = Net::SSH.start(@node.ip, @node.auth[:username], make_ssh_opts)
+      connected?
+    end
+
+    def connected?
+      @ssh && (not @ssh.closed?)
+    end
+
+    def make_ssh_opts
+      ssh_opts = {
+        number_of_password_prompts:      0,
+        keepalive:                       vars(:ssh_no_keepalive) ? false : true,
+        verify_host_key:                 must_secure? ? :always : :never,
+        append_all_supported_algorithms: true,
+        password:                        @node.auth[:password],
+        timeout:                         @node.timeout,
+        port:                            (vars(:ssh_port) || 22).to_i,
+        forward_agent:                   false
+      }
+
+      auth_methods = vars(:auth_methods) || %w[none publickey password]
+      ssh_opts[:auth_methods] = auth_methods
+      logger.debug "AUTH METHODS::#{auth_methods}"
+
+      if vars(:ssh_proxy)
+        ssh_opts[:proxy] = make_ssh_proxy_command(
+          vars(:ssh_proxy), vars(:ssh_proxy_port), must_secure?
+        )
+      end
+      ssh_opts[:keys]       = [vars(:ssh_keys)].flatten           if vars(:ssh_keys)
+      ssh_opts[:kex]        = vars(:ssh_kex).split(/,\s*/)        if vars(:ssh_kex)
+      ssh_opts[:encryption] = vars(:ssh_encryption).split(/,\s*/) if vars(:ssh_encryption)
+      ssh_opts[:host_key]   = vars(:ssh_host_key).split(/,\s*/)   if vars(:ssh_host_key)
+      ssh_opts[:hmac]       = vars(:ssh_hmac).split(/,\s*/)       if vars(:ssh_hmac)
+
+      # Use our logger for Net::SSH
+      ssh_logger = SemanticLogger[Net::SSH]
+      config_debug = Oxidized.config.input.debug
+      if config_debug == true ||
+         (config_debug.is_a?(String) && config_debug.downcase.include?('library'))
+        ssh_logger.level = :debug
+      else
+        ssh_logger.level = :fatal
+      end
+      ssh_opts[:logger] = ssh_logger
+
+      ssh_opts
+    end
+
+    def must_secure?
+      Oxidized.config.input[config_name].secure? == true
+    end
+
+    def make_ssh_proxy_command(proxy_host, proxy_port, secure)
+      return nil unless !proxy_host.nil? && !proxy_host.empty?
+
+      proxy_command =  "ssh "
+      proxy_command += "-o StrictHostKeyChecking=no " unless secure
+      proxy_command += "-p #{proxy_port} "            if proxy_port
+      proxy_command += "#{proxy_host} -W [%h]:%p"
+      Net::SSH::Proxy::Command.new(proxy_command)
+    end
+
+    def disconnect
+      disconnect_cli
+      Timeout.timeout(@node.timeout) do
+        @ssh.close
+      end
+    rescue Errno::ECONNRESET, Net::SSH::Disconnect, IOError => e
+      logger.debug 'The other side closed the connection while ' \
+                   "disconnecting, raising #{e.class} with #{e.message}"
+    rescue Timeout::Error
+      logger.debug "#{@node.name} timed out while disconnecting"
+    end
+
+    # Methods to implement in subclasses
+    def cmd(**_args)
+      raise NotImplementedError, "Subclasses must implement cmd"
+    end
+  end
+end

data/lib/oxidized/input/telnet.rb

@@ -1,16 +1,17 @@
+require_relative 'debugtext'
+
 module Oxidized
   require 'net/telnet'
-  require 'oxidized/input/cli'
   class Telnet < Input
-    RescueFail = {}.freeze
-    include Input::CLI
     attr_reader :telnet
 
-    def connect(node)
+    def connect(node) # rubocop:disable Naming/PredicateMethod
       @node    = node
-      @timeout = Oxidized.config.timeout
+      @timeout = @node.timeout
       @node.model.cfg['telnet'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-telnet", 'w') if Oxidized.config.input.debug?
+
+      @text_debug = DebugText.new(Oxidized.config.input.debug, @node, config_name)
+
       port = vars(:telnet_port) || 23
 
       telnet_opts = {
@@ -18,7 +19,7 @@ module Oxidized
         'Port'    => port.to_i,
         'Timeout' => @timeout,
         'Model'   => @node.model,
-        'Log'     => @log
+        'Log'     => @text_debug
       }
 
       @telnet = Net::Telnet.new telnet_opts
@@ -35,18 +36,20 @@ module Oxidized
     end
 
     def cmd(cmd_str, expect = @node.prompt)
-      Oxidized.logger.debug "Telnet: #{cmd_str} @#{@node.name}"
+      logger.debug "Telnet: #{cmd_str} @#{@node.name}"
       return send(cmd_str + "\r\n") unless expect
 
       # create a string to be passed to oxidized_expect and modified _there_
       # default to a single space so it shouldn't be coerced to nil by any models.
       out = String(' ')
+      @text_debug&.send_data(cmd_str)
       @telnet.puts(cmd_str)
       @telnet.oxidized_expect(timeout: @timeout, expect: expect, out: out)
       out
     end
 
     def send(data)
+      @text_debug&.send_data(data)
       @telnet.write data
     end
 
@@ -63,10 +66,11 @@ module Oxidized
     def disconnect
       disconnect_cli
       @telnet.close
-    rescue Errno::ECONNRESET
+    rescue Errno::ECONNRESET, IOError
+      # This exception is intented and therefore not handled here
     ensure
-      @log.close if Oxidized.config.input.debug?
-      (@telnet.close rescue true) unless @telnet.sock.closed?
+      @text_debug&.close
+      (@telnet.close rescue true) unless @telnet.sock.closed? # rubocop:disable Style/RedundantParentheses
     end
   end
 end
@@ -77,12 +81,12 @@ module Net
     ## FIXME: we also need output (not sure I'm going to support this)
     attr_reader :output
 
-    def oxidized_expect(options) ## rubocop:disable Metrics/PerceivedComplexity
-      model    = @options["Model"]
-      @log     = @options["Log"]
+    def oxidized_expect(options)
+      model      = @options["Model"]
+      text_debug = @options["Log"]
 
       expects  = [options[:expect]].flatten
-      time_out = options[:timeout] || @options["Timeout"] || Oxidized.config.timeout?
+      time_out = options[:timeout] || @options["Timeout"]
 
       Timeout.timeout(time_out) do
         line = ""
@@ -105,10 +109,7 @@ module Net
             buf = preprocess(c)
             rest = ''
           end
-          if Oxidized.config.input.debug?
-            @log.print buf
-            @log.flush
-          end
+          text_debug&.receive_data buf
           line += buf
           line = model.expects line
           # match is a regexp object. we need to return that for logins to work.

data/lib/oxidized/input/tftp.rb

@@ -1,7 +1,5 @@
 module Oxidized
   require 'stringio'
-  require_relative 'cli'
-
   begin
     require 'net/tftp'
   rescue LoadError
@@ -9,19 +7,24 @@ module Oxidized
   end
 
   class TFTP < Input
-    include Input::CLI
+    RESCUE_FAIL = {
+      Net::TFTPTimeout => :warn
+    }.freeze
+
+    def self.rescue_fail
+      super.merge(RESCUE_FAIL)
+    end
 
     # TFTP utilizes UDP, there is not a connection. We simply specify an IP and send/receive data.
     def connect(node)
       @node = node
 
       @node.model.cfg['tftp'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-tftp", 'w') if Oxidized.config.input.debug?
       @tftp = Net::TFTP.new @node.ip
     end
 
     def cmd(file)
-      Oxidized.logger.debug "TFTP: #{file} @ #{@node.name}"
+      logger.debug "TFTP: #{file} @ #{@node.name}"
       config = StringIO.new
       @tftp.getbinary file, config
       config.rewind
@@ -32,9 +35,6 @@ module Oxidized
 
     def disconnect
       # TFTP uses UDP, there is no connection to close
-      true
-    ensure
-      @log.close if Oxidized.config.input.debug?
     end
   end
 end

data/lib/oxidized/job.rb

@@ -1,16 +1,27 @@
 module Oxidized
   class Job < Thread
+    include SemanticLogger::Loggable
+
     attr_reader :start, :end, :status, :time, :node, :config
 
     def initialize(node)
-      @node         = node
-      @start        = Time.now.utc
+      @node = node
+      @start = Time.now.utc
+      self.name = "Job '#{@node.name}'"
       super do
-        Oxidized.logger.debug "lib/oxidized/job.rb: Starting fetching process for #{@node.name} at #{Time.now.utc}"
-        @status, @config = @node.run
-        @end             = Time.now.utc
-        @time            = @end - @start
-        Oxidized.logger.debug "lib/oxidized/job.rb: Config fetched for #{@node.name} at #{@end}"
+        logger.debug "Starting fetching process for #{@node.name}"
+        begin
+          Timeout.timeout(Oxidized.config.timelimit) do
+            @status, @config = @node.run
+          end
+          logger.debug "Config fetched for #{@node.name}"
+        rescue Timeout::Error
+          logger.warn "Job timelimit reached for #{@node.name}"
+          @status = :timelimit
+        ensure
+          @end  = Time.now.utc
+          @time = @end - @start
+        end
       end
     end
   end