oxidized 0.35.0 → 0.37.0

data/lib/oxidized/cli/support.rb

@@ -0,0 +1,152 @@
+require 'time'
+
+module Oxidized
+  class CLI
+    module Support
+      SENSITIVE_NAME_RE = /(password|passphrase|secret|token|enable|
+                            (private|api|access)_?key|
+                            community|credential|auth
+                          )/ix
+      ROOT_GEMS = %w[oxidized oxidized-web].freeze
+      EXPLICIT_ENV_KEYS = %w[
+        OXIDIZED_HOME
+        OXIDIZED_LOGS
+        CONFIG_RELOAD_INTERVAL
+        UPDATE_CA_CERTIFICATES
+      ].freeze
+
+      private
+
+      def show_support_details
+        print_intro
+        print_environment
+        print_config_files
+        print_rugged_support
+        print_installed_gems
+      end
+
+      def print_intro
+        os_release = read_os_release
+        runit_path = '/etc/service/oxidized/run'
+
+        puts '> :warning:'
+        puts '> The --support option is intended for diagnostic purposes and may include sensitive information.'
+        puts '> Remove any sensitive data before sharing this output.'
+        puts
+        puts '## Oxidized Support Data'
+        puts "- Timestamp: #{Time.now.utc.iso8601}"
+        puts "- Oxidized version: #{Oxidized::VERSION_FULL}"
+        puts "- OS release: #{os_release}" if os_release
+        puts "- Container hint (#{runit_path} exists): #{File.exist?(runit_path)}"
+        puts "- Ruby engine: #{defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby'}"
+        puts "- Ruby version: #{RUBY_VERSION}p#{RUBY_PATCHLEVEL} (#{RUBY_PLATFORM})"
+        puts "- Working directory: #{Dir.pwd}"
+        puts "- Gem paths: #{Gem.path.join(', ')}"
+        puts
+      end
+
+      def print_environment
+        puts '### Environment Variables'
+        keys = (ENV.keys.grep(/^OXIDIZED_/) + EXPLICIT_ENV_KEYS).uniq.sort
+
+        keys.each do |key|
+          next unless ENV.has_key?(key)
+
+          value = ENV.fetch(key)
+          puts key.match?(SENSITIVE_NAME_RE) ? "#{key}=[REDACTED]" : "#{key}=#{value}"
+        end
+
+        puts
+      end
+
+      def print_config_files
+        puts '### Configuration Files'
+        config_paths.each do |path|
+          config_path = File.expand_path(path)
+          exists = File.exist?(config_path)
+          puts "- #{config_path} exists: #{exists ? 'yes' : 'no'}"
+          next unless exists
+
+          print_sanitized_config(config_path)
+        end
+        puts
+      end
+
+      def print_rugged_support
+        puts '### Rugged'
+        begin
+          require 'rugged'
+          puts "- Rugged version: #{Rugged::VERSION}"
+
+          ssh_supported = Rugged.respond_to?(:features) && Rugged.features.include?(:ssh)
+          puts "- Rugged SSH support: #{ssh_supported}"
+        rescue LoadError
+          puts '- Rugged: not available'
+          puts '- Rugged SSH support: false'
+        end
+        puts
+      end
+
+      def print_installed_gems
+        puts '### Relevant Installed Gems'
+        relevant_gem_names.each do |name|
+          versions = Gem::Specification.find_all_by_name(name).sort_by(&:version).map { |s| s.version.to_s }
+          puts "- #{name} (#{versions.join(', ')})" unless versions.empty?
+        end
+      end
+
+      def relevant_gem_names
+        names = ROOT_GEMS.select { |name| Gem::Specification.any? { |s| s.name == name } }
+
+        root_specs = names.flat_map { |name| Gem::Specification.find_all_by_name(name) }
+        runtime_deps = root_specs.flat_map { |spec| spec.dependencies }
+        names.concat(runtime_deps.map(&:name))
+        names.sort.uniq
+      end
+
+      def config_paths
+        user_default = File.join(Dir.home, '.config', 'oxidized', 'config')
+        home_from_env = File.join(File.expand_path(Oxidized::Config::ROOT), 'config')
+
+        [
+          '/etc/oxidized/config',
+          user_default,
+          home_from_env
+        ].uniq
+      end
+
+      def print_sanitized_config(path)
+        content = File.read(path)
+        puts '```yaml'
+        content.each_line(chomp: true) do |line|
+          key, separator, = line.partition(':')
+
+          if separator.empty? || key.empty?
+            puts line
+            next
+          end
+
+          if key.match?(SENSITIVE_NAME_RE)
+            puts "#{key}: [REDACTED]"
+          else
+            puts line
+          end
+        end
+        puts '```'
+      rescue StandardError => e
+        puts "    <failed to read: #{e.class}: #{e.message}>"
+      end
+
+      def read_os_release
+        return nil unless File.exist?('/etc/os-release')
+
+        line = File.foreach('/etc/os-release').find { |entry| entry.start_with?('PRETTY_NAME=') }
+        return nil unless line
+
+        line.split('=', 2).last.to_s.strip.gsub(/^"|"$/, '')
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/oxidized/cli.rb

@@ -1,8 +1,10 @@
 require 'semantic_logger'
+require_relative 'cli/support'
 
 module Oxidized
   class CLI
     include SemanticLogger::Loggable
+    include Support
 
     require 'slop'
     require 'oxidized'
@@ -48,6 +50,7 @@ module Oxidized
       opts = Slop.parse do |opt|
         opt.on '-d', '--debug', 'turn on debugging'
         opt.on '--daemonize', 'Daemonize/fork the process'
+        opt.on '--support', 'show support diagnostics and exit'
         opt.string '--home-dir', 'Oxidized home dir', default: nil
         opt.string '--config-file', 'Oxidized config file', default: nil
         opt.on '-h', '--help', 'show usage' do
@@ -64,6 +67,12 @@ module Oxidized
           Kernel.exit
         end
       end
+
+      if opts[:support]
+        show_support_details
+        Kernel.exit
+      end
+
       [opts.arguments, opts]
     end
 

data/lib/oxidized/hook/githubrepo.rb

@@ -86,7 +86,7 @@ class GithubRepo < Oxidized::Hook
   private
 
   def credentials(node)
-    Proc.new do |_url, username_from_url, _allowed_types| # rubocop:disable Style/Proc
+    proc do |_url, username_from_url, _allowed_types|
       git_user = cfg.has_key?('username') ? cfg.username : (username_from_url || 'git')
       if cfg.has_key?('password')
         logger.debug "Authenticating using username and password as '#{git_user}'"
@@ -96,6 +96,7 @@ class GithubRepo < Oxidized::Hook
         logger.debug "Authenticating using ssh keys as '#{git_user}'"
         rugged_sshkey(git_user: git_user, privkey: cfg.privatekey, pubkey: pubkey)
       elsif cfg.has_key?('remote_repo') &&
+            cfg.remote_repo.respond_to?(:has_key?) &&
             cfg.remote_repo.has_key?(node.group) &&
             cfg.remote_repo[node.group].has_key?('privatekey')
         pubkey = cfg.remote_repo[node.group].has_key?('publickey') ? cfg.remote_repo[node.group].publickey : nil

data/lib/oxidized/hook.rb

@@ -6,6 +6,8 @@ module Oxidized
       def from_config(cfg)
         mgr = new
         cfg.hooks.each do |name, h_cfg|
+          raise("Please specify an hook type in the configuration") unless h_cfg.type?
+
           h_cfg.events.each do |event|
             mgr.register event.to_sym, name, h_cfg.type, h_cfg
           end
@@ -14,11 +16,14 @@ module Oxidized
       end
     end
 
-    # HookContext is passed to each hook. It can contain anything related to the
-    # event in question. At least it contains the event name
-    # The argument keyword_init: true is needed for ruby < 3.2 and can be
-    # dropped with the support of ruby 3.1
-    HookContext = Struct.new(:event, :node, :job, :commitref, keyword_init: true)
+    # HookContext is passed to each hook. It always carries the event name.
+    # The keyword_init: true argument forces keyword-argument initialization.
+    HookContext = Struct.new(
+      :event, :node, :job, :commitref,
+      :node_raw,   # raw source record: JSON hash, SQL row hash, CSV field array
+      :context,    # self from call site, to access methods/bindings of call site
+      keyword_init: true
+    )
 
     # RegisteredHook is a container for a Hook instance
     RegisteredHook = Struct.new(:name, :hook)
@@ -28,6 +33,7 @@ module Oxidized
       node_fail
       post_store
       nodes_done
+      source_node_transform
     ].freeze
     attr_reader :registered_hooks
 
@@ -54,16 +60,60 @@ module Oxidized
       logger.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}"
     end
 
+    # --- Transform events ---
+
+    # Runs source_node_transform hooks in sequence, passing the return value of
+    # each hook as node to the next. Returns the final node, or nil
+    # to signal that the node should be excluded.
+    def source_node_transform(node:, node_raw:, context:)
+      ctx = HookContext.new(
+        event:    :source_node_transform,
+        node:     node,
+        node_raw: node_raw,
+        context:  context
+      )
+      @registered_hooks[:source_node_transform].each do |r_hook|
+        ctx.node = r_hook.hook.run_hook(ctx)
+      rescue StandardError => e
+        logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
+                     "(#{e.inspect}) for event :source_node_transform"
+      end
+      ctx.node
+    end
+
+    # --- Fire-and-forget events ---
+
+    def node_success(node:, job: nil)
+      handle(:node_success, node: node, job: job)
+    end
+
+    def node_fail(node:, job: nil)
+      handle(:node_fail, node: node, job: job)
+    end
+
+    def post_store(node:, job: nil, commitref: nil)
+      handle(:post_store, node: node, job: job, commitref: commitref)
+    end
+
+    def nodes_done
+      handle(:nodes_done)
+    end
+
+    private
+
+    # Shared implementation for fire-and-forget events: runs all registered
+    # hooks for the event, ignores return values, logs errors.
     def handle(event, ctx_params = {})
-      ctx = HookContext.new ctx_params
-      ctx.event = event
+      ctx = HookContext.new(event: event, **ctx_params)
 
       @registered_hooks[event].each do |r_hook|
-        r_hook.hook.run_hook ctx
+        r_hook.hook.run_hook(ctx)
       rescue StandardError => e
         logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
                      "(#{e.inspect}) for event #{event.inspect}"
       end
+
+      nil
     end
   end
 

data/lib/oxidized/input/debugtext.rb

@@ -0,0 +1,40 @@
+module Oxidized
+  class DebugText
+    include SemanticLogger::Loggable
+
+    def initialize(config_debug, node, input_name)
+      return unless config_debug == true ||
+                    (config_debug.is_a?(String) && config_debug.downcase.include?('text'))
+
+      @log = File.open(logfile(node, input_name), 'w')
+    end
+
+    # Separate method to ease unit tests
+    def logfile(node, input_name)
+      timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
+      file = Oxidized::Config::LOG + "/#{node&.ip}-#{input_name}-#{timestamp}.txt"
+      logger.debug "Writing I/O Debugging to #{file}"
+      file
+    end
+
+    def send_data(data)
+      return unless @log
+
+      @log.puts "sent cmd #{data.dump}"
+      @log.flush
+    end
+
+    def receive_data(data)
+      return unless @log
+
+      @log.puts "received #{data.dump}"
+      @log.flush
+    end
+
+    def close
+      return unless @log
+
+      @log.close
+    end
+  end
+end

data/lib/oxidized/input/debugyaml.rb

@@ -0,0 +1,82 @@
+module Oxidized
+  class DebugYAML
+    include SemanticLogger::Loggable
+
+    def initialize(config_debug, node, input_name)
+      return unless config_debug == true ||
+                    (config_debug.is_a?(String) && config_debug.downcase.include?('yaml'))
+
+      @log = File.open(logfile(node, input_name), 'w')
+
+      @partial_line = false
+      @first_line = true
+      @commands_started = false
+
+      @log.puts '---'
+      @log.puts 'init_prompt: |-'
+      @log.flush
+    end
+
+    # Separate method to ease unit tests
+    def logfile(node, input_name)
+      timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
+      file = Oxidized::Config::LOG + "/#{node&.ip}-#{input_name}-#{timestamp}.yaml"
+      logger.debug "Writing YAML Simulation to #{file}"
+      file
+    end
+
+    def send_data(data)
+      return unless @log
+
+      @log.puts
+      @log.puts 'commands:' unless @commands_started
+      @log.puts "  - #{data.dump}: |-"
+      @first_line = true
+      @partial_line = false
+      @commands_started = true
+      @log.flush
+    end
+
+    def receive_data(data)
+      return unless @log
+      return if data.empty?
+
+      lines = data.split("\n", -1)
+
+      lines.each_with_index do |line, idx|
+        is_last = idx == lines.length - 1
+        full_line = is_last ? (data[-1] == "\n") : true
+        # Escape line and strip surrounding double quotes
+        line = line.dump[1..-2]
+        if @first_line
+          # Make sure the leading space of the first line (if present)
+          # is coded with \0x20 or YAML block scalars won't work
+          line.sub!(/^ /, '\x20')
+          @first_line = false
+        end
+
+        # Make sure trailing white spaces are coded with \0x20
+        line.gsub!(/ $/, '\x20')
+
+        output = @partial_line ? line : ('      ' + line)
+        @partial_line = false
+
+        if full_line
+          @log.puts output
+        else
+          @log.write output
+        end
+      end
+
+      @partial_line = data[-1] != "\n"
+
+      @log.flush
+    end
+
+    def close
+      return unless @log
+
+      @log.close
+    end
+  end
+end

data/lib/oxidized/input/exec.rb

@@ -1,12 +1,7 @@
 module Oxidized
-  require "oxidized/input/cli"
-
   class Exec < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
-      @log = File.open(Oxidized::Config::LOG + "/#{@node.ip}-exec", "w") if Oxidized.config.input.debug?
       @node.model.cfg["exec"].each { |cb| instance_exec(&cb) }
     end
 
@@ -19,10 +14,6 @@ module Oxidized
 
     private
 
-    def disconnect
-      true
-    ensure
-      @log.close if Oxidized.config.input.debug?
-    end
+    def disconnect; end
   end
 end

data/lib/oxidized/input/ftp.rb

@@ -1,24 +1,10 @@
 module Oxidized
   require 'net/ftp'
   require 'timeout'
-  require_relative 'cli'
-
   class FTP < Input
-    RESCUE_FAIL = {
-      debug: [
-        # Net::SSH::Disconnect,
-      ],
-      warn:  [
-        # RuntimeError,
-        # Net::SSH::AuthenticationFailed,
-      ]
-    }.freeze
-    include Input::CLI
-
     def connect(node) # rubocop:disable Naming/PredicateMethod
       @node = node
       @node.model.cfg['ftp'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::LOG + "/#{@node.ip}-ftp", 'w') if Oxidized.config.input.debug?
       @ftp = Net::FTP.new(@node.ip)
       @ftp.passive = Oxidized.config.input.ftp.passive
       @ftp.login @node.auth[:username], @node.auth[:password]
@@ -47,9 +33,6 @@ module Oxidized
 
     def disconnect
       @ftp.close
-    # rescue Errno::ECONNRESET, IOError
-    ensure
-      @log.close if Oxidized.config.input.debug?
     end
   end
 end

data/lib/oxidized/input/http.rb

@@ -1,19 +1,15 @@
 module Oxidized
-  require "oxidized/input/cli"
   require "net/http"
   require "json"
   require "net/http/digest_auth"
 
   class HTTP < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
       @secure = false
       @username = nil
       @password = nil
       @headers = {}
-      @log = File.open(Oxidized::Config::LOG + "/#{@node.ip}-http", "w") if Oxidized.config.input.debug?
       @node.model.cfg["http"].each { |cb| instance_exec(&cb) }
 
       return true unless @main_page && defined?(login)
@@ -48,50 +44,72 @@ module Oxidized
     private
 
     def get_http(path)
+      res = perform_http_request(path, method: :get)
+      res.body
+    end
+
+    def post_http(path, body = nil, extra_headers = {})
+      res = perform_http_request(path, method: :post, body: body, extra_headers: extra_headers)
+      res.body
+    end
+
+    def perform_http_request(path, method: :get, body: nil, extra_headers: {})
       uri = get_uri(path)
+      http_method = method.to_s.upcase
 
-      logger.debug "Making request to: #{uri}"
+      logger.debug "Making #{http_method} request to: #{uri}"
 
       ssl_verify = Oxidized.config.input.http.ssl_verify? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
 
-      res = make_request(uri, ssl_verify)
+      res = make_request(uri, ssl_verify, extra_headers, method: method, body: body)
 
       if res.code == '401' && res['www-authenticate']&.include?('Digest')
         uri.user = @username
         uri.password = URI.encode_www_form_component(@password)
         logger.debug "Server requires Digest authentication"
-        auth = Net::HTTP::DigestAuth.new.auth_header(uri, res['www-authenticate'], 'GET')
 
-        res = make_request(uri, ssl_verify, 'Authorization' => auth)
-      elsif @username && @password
+        auth = Net::HTTP::DigestAuth.new.auth_header(uri, res['www-authenticate'], http_method)
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => auth),
+                           method: method, body: body)
+
+      elsif @username && @password && !authorization_header_present?(extra_headers)
         logger.debug "Falling back to Basic authentication"
-        res = make_request(uri, ssl_verify, 'Authorization' => basic_auth_header)
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => basic_auth_header),
+                           method: method, body: body)
       end
 
       logger.debug "Response code: #{res.code}"
-      res.body
+      res
     end
 
-    def make_request(uri, ssl_verify, extra_headers = {})
+    def make_request(uri, ssl_verify, extra_headers = {}, method: :get, body: nil)
       Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https", verify_mode: ssl_verify) do |http|
-        req = Net::HTTP::Get.new(uri)
+        req_class = if method == :get
+                      Net::HTTP::Get
+                    elsif method == :post
+                      Net::HTTP::Post
+                    else
+                      raise Oxidized::OxidizedError, "Unsupported HTTP method: #{method.inspect}. " \
+                                                     "Only :get and :post are supported"
+                    end
+        req = req_class.new(uri)
         @headers.merge(extra_headers).each { |header, value| req.add_field(header, value) }
-        logger.debug "Sending request with headers: #{@headers.merge(extra_headers)}"
+        req.body = body if body
+
+        logger.debug "Sending #{method.to_s.upcase} request with headers: #{@headers.merge(extra_headers)}"
         http.request(req)
       end
     end
 
-    def basic_auth_header
-      "Basic " + ["#{@username}:#{@password}"].pack('m').delete("\r\n")
+    def authorization_header_present?(headers)
+      headers.keys.any? { |key| key.to_s.downcase == 'authorization' }
     end
 
-    def log(str)
-      @log&.write(str)
+    def basic_auth_header
+      "Basic " + ["#{@username}:#{@password}"].pack('m').delete("\r\n")
     end
 
-    def disconnect
-      @log.close if Oxidized.config.input.debug?
-    end
+    def disconnect; end
 
     def get_uri(path)
       path = URI.parse(path)

data/lib/oxidized/input/input.rb

@@ -1,24 +1,44 @@
+require_relative 'cli'
+
 module Oxidized
   class PromptUndetect < OxidizedError; end
 
   class Input
     include SemanticLogger::Loggable
     include Oxidized::Config::Vars
+    include Oxidized::Input::CLI
 
     RESCUE_FAIL = {
-      debug: [
-        Errno::ECONNREFUSED
-      ],
-      warn:  [
-        IOError,
-        PromptUndetect,
-        Timeout::Error,
-        Errno::ECONNRESET,
-        Errno::EHOSTUNREACH,
-        Errno::ENETUNREACH,
-        Errno::EPIPE,
-        Errno::ETIMEDOUT
-      ]
+      Errno::ECONNREFUSED => :debug,
+      IOError             => :warn,
+      PromptUndetect      => :warn,
+      Timeout::Error      => :warn,
+      Errno::ECONNRESET   => :warn,
+      Errno::EHOSTUNREACH => :warn,
+      Errno::ENETUNREACH  => :warn,
+      Errno::EPIPE        => :warn,
+      Errno::ETIMEDOUT    => :warn
     }.freeze
+
+    # Returns a hash mapping exception classes to their log level
+    def self.rescue_fail
+      RESCUE_FAIL.dup
+    end
+
+    def self.config_name
+      name.split('::').last.downcase
+    end
+
+    def self.to_sym
+      config_name.to_sym
+    end
+
+    def config_name
+      self.class.config_name
+    end
+
+    def to_sym
+      self.class.to_sym
+    end
   end
 end