oxidized 0.29.1 → 0.37.0

data/lib/oxidized/hook/githubrepo.rb

@@ -1,20 +1,30 @@
+require 'rugged'
+
 class GithubRepo < Oxidized::Hook
   def validate_cfg!
     raise KeyError, 'hook.remote_repo is required' unless cfg.has_key?('remote_repo')
   end
 
   def run_hook(ctx)
+    unless ctx.node
+      logger.error 'GithubRepo.run_hook: no node provided'
+      return
+    end
+
+    unless ctx.node.repo
+      logger.error "Oxidized output is not git, can't push to remote"
+      return
+    end
     repo  = Rugged::Repository.new(ctx.node.repo)
     creds = credentials(ctx.node)
     url   = remote_repo(ctx.node)
 
     if url.nil? || url.empty?
-      log "No repository defined for #{ctx.node.group}/#{ctx.node.name}", :debug
+      logger.error "No repository defined for #{ctx.node.group}/#{ctx.node.name}"
       return
     end
 
-    log "Pushing local repository(#{repo.path})..."
-    log "to remote: #{url}"
+    logger.info "Pushing local repository(#{repo.path}) to remote: #{url}"
 
     if repo.remotes['origin'].nil?
       repo.remotes.create('origin', url)
@@ -23,28 +33,46 @@ class GithubRepo < Oxidized::Hook
     end
     remote = repo.remotes['origin']
 
-    fetch_and_merge_remote(repo, creds)
-
-    remote.push([repo.head.name], credentials: creds)
+    begin
+      fetch_and_merge_remote(repo, creds)
+      remote.push([repo.head.name], credentials: creds)
+    rescue Rugged::NetworkError => e
+      if e.message == 'unsupported URL protocol'
+        logger.warn "Rugged does not support the git URL '#{url}'."
+        unless Rugged.features.include?(:ssh)
+          logger.warn "Note: Rugged isn't installed with ssh support. You may need " \
+                      '"gem install rugged -- --with-ssh"'
+        end
+      end
+      # re-raise exception for the calling method
+      raise
+    end
   end
 
   def fetch_and_merge_remote(repo, creds)
     result = repo.fetch('origin', [repo.head.name], credentials: creds)
-    log result.inspect, :debug
+    logger.debug result.inspect
+
+    their_branch = remote_branch(repo)
 
-    unless result[:total_deltas].positive?
-      log "nothing received after fetch", :debug
+    unless their_branch
+      logger.debug 'remote branch does not exist yet, nothing to merge'
       return
     end
 
-    their_branch = repo.branches["origin/master"]
+    result = repo.merge_analysis(their_branch.target_id)
 
-    log "merging fetched branch #{their_branch.name}", :debug
+    if result.include? :up_to_date
+      logger.debug 'nothing to merge'
+      return
+    end
+
+    logger.debug "merging fetched branch #{their_branch.name}"
 
     merge_index = repo.merge_commits(repo.head.target_id, their_branch.target_id)
 
     if merge_index.conflicts?
-      log("Conflicts detected, skipping Rugged::Commit.create", :warn)
+      logger.warn "Conflicts detected, skipping Rugged::Commit.create"
       return
     end
 
@@ -58,21 +86,24 @@ class GithubRepo < Oxidized::Hook
   private
 
   def credentials(node)
-    Proc.new do |_url, username_from_url, _allowed_types| # rubocop:disable Style/Proc
+    proc do |_url, username_from_url, _allowed_types|
       git_user = cfg.has_key?('username') ? cfg.username : (username_from_url || 'git')
       if cfg.has_key?('password')
-        log "Authenticating using username and password as '#{git_user}'", :debug
+        logger.debug "Authenticating using username and password as '#{git_user}'"
         Rugged::Credentials::UserPassword.new(username: git_user, password: cfg.password)
       elsif cfg.has_key?('privatekey')
         pubkey = cfg.has_key?('publickey') ? cfg.publickey : nil
-        log "Authenticating using ssh keys as '#{git_user}'", :debug
+        logger.debug "Authenticating using ssh keys as '#{git_user}'"
         rugged_sshkey(git_user: git_user, privkey: cfg.privatekey, pubkey: pubkey)
-      elsif cfg.has_key?('remote_repo') && cfg.remote_repo.has_key?(node.group) && cfg.remote_repo[node.group].has_key?('privatekey')
+      elsif cfg.has_key?('remote_repo') &&
+            cfg.remote_repo.respond_to?(:has_key?) &&
+            cfg.remote_repo.has_key?(node.group) &&
+            cfg.remote_repo[node.group].has_key?('privatekey')
         pubkey = cfg.remote_repo[node.group].has_key?('publickey') ? cfg.remote_repo[node.group].publickey : nil
-        log "Authenticating using ssh keys as '#{git_user}' for '#{node.group}/#{node.name}'", :debug
+        logger.debug "Authenticating using ssh keys as '#{git_user}' for '#{node.group}/#{node.name}'"
         rugged_sshkey(git_user: git_user, privkey: cfg.remote_repo[node.group].privatekey, pubkey: pubkey)
       else
-        log "Authenticating using ssh agent as '#{git_user}'", :debug
+        logger.debug "Authenticating using ssh agent as '#{git_user}'"
         Rugged::Credentials::SshKeyFromAgent.new(username: git_user)
       end
     end
@@ -97,4 +128,10 @@ class GithubRepo < Oxidized::Hook
       cfg.remote_repo[node.group].url
     end
   end
+
+  # Returns a Rugged::Branch to the remote branch or nil if it doen't exist
+  def remote_branch(repo)
+    head_branch = repo.branches[repo.head.name]
+    repo.branches['origin/' + head_branch.name]
+  end
 end

data/lib/oxidized/hook/noophook.rb

@@ -1,9 +1,9 @@
 class NoopHook < Oxidized::Hook
   def validate_cfg!
-    log "Validate config"
+    logger.info "Validate config"
   end
 
   def run_hook(ctx)
-    log "Run hook with context: #{ctx}"
+    logger.info "Run hook with context: #{ctx}"
   end
 end

data/lib/oxidized/hook/slackdiff.rb

@@ -1,4 +1,6 @@
-require 'slack'
+require 'slack_ruby_client'
+require 'uri'
+require 'net/http'
 
 # defaults to posting a diff, if messageformat is supplied them a message will be posted too
 # diff defaults to true
@@ -9,39 +11,73 @@ class SlackDiff < Oxidized::Hook
     raise KeyError, 'hook.channel is required' unless cfg.has_key?('channel')
   end
 
+  def slack_upload(client, title, content, channel, proxy)
+    logger.info "Posting diff as snippet to #{channel}"
+    upload_dest = client.files_getUploadURLExternal(filename:     "change",
+                                                    length:       content.length,
+                                                    snippet_type: "diff")
+    file_uri = URI.parse(upload_dest[:upload_url])
+
+    proxy_uri = URI.parse(proxy) if proxy
+    proxy_address = proxy_uri ? proxy_uri.host : :ENV
+    proxy_port = proxy_uri&.port
+    proxy_user = proxy_uri&.user
+    proxy_pass = proxy_uri&.password
+
+    http = Net::HTTP.new(file_uri.host, file_uri.port, proxy_address, proxy_port, proxy_user, proxy_pass)
+    http.use_ssl = true
+
+    request = Net::HTTP::Post.new(file_uri.request_uri, { Host: file_uri.host })
+    request.body = content
+    response = http.request(request)
+
+    raise 'Slack file upload failed' unless response.is_a? Net::HTTPSuccess
+
+    files = [{
+      id:    upload_dest[:file_id],
+      title: title
+    }]
+    begin
+      client.files_completeUploadExternal(channel_id: channel,
+                                          files:      files.to_json)
+    rescue Slack::Web::Api::Errors::NotInChannel
+      logger.info "Not in specified channel, attempting to join"
+      client.conversations_join(channel: channel)
+      client.files_completeUploadExternal(channel_id: channel,
+                                          files:      files.to_json)
+    end
+  end
+
   def run_hook(ctx)
     return unless ctx.node
     return unless ctx.event.to_s == "post_store"
 
-    log "Connecting to slack"
+    logger.info "Connecting to slack"
     Slack::Web::Client.configure do |config|
       config.token = cfg.token
       config.proxy = cfg.proxy if cfg.has_key?('proxy')
     end
     client = Slack::Web::Client.new
     client.auth_test
-    log "Connected"
+    logger.info "Connected"
     if cfg.has_key?("diff") ? cfg.diff : true
       gitoutput = ctx.node.output.new
       diff = gitoutput.get_diff ctx.node, ctx.node.group, ctx.commitref, nil
       unless diff == "no diffs"
         title = "#{ctx.node.name} #{ctx.node.group} #{ctx.node.model.class.name.to_s.downcase}"
-        log "Posting diff as snippet to #{cfg.channel}"
-        client.files_upload(channels: cfg.channel, as_user: true,
-                            content: diff[:patch].lines.to_a[4..-1].join,
-                            filetype: "diff",
-                            title: title,
-                            filename: "change")
+        content = diff[:patch].lines.to_a[4..-1].join
+        slack_upload(client, title, content, cfg.channel, cfg.has_key?('proxy') ? cfg.proxy : nil)
       end
     end
     # message custom formatted - optional
     if cfg.message?
-      log cfg.message
-      msg = cfg.message % { node: ctx.node.name.to_s, group: ctx.node.group.to_s, commitref: ctx.commitref, model: ctx.node.model.class.name.to_s.downcase }
-      log msg
-      log "Posting message to #{cfg.channel}"
+      logger.info cfg.message
+      msg = cfg.message % { node: ctx.node.name.to_s, group: ctx.node.group.to_s, commitref: ctx.commitref,
+                            model: ctx.node.model.class.name.to_s.downcase }
+      logger.info msg
+      logger.info "Posting message to #{cfg.channel}"
       client.chat_postMessage(channel: cfg.channel, text: msg, as_user: true)
     end
-    log "Finished"
+    logger.info "Finished"
   end
 end

data/lib/oxidized/hook/xmppdiff.rb

@@ -5,35 +5,35 @@ class XMPPDiff < Oxidized::Hook
   def connect
     @client = Jabber::Client.new(Jabber::JID.new(cfg.jid))
 
-    log "Connecting to XMPP"
+    logger.info "Connecting to XMPP"
     begin
       Timeout.timeout(15) do
         begin
           @client.connect
         rescue StandardError => e
-          log "Failed to connect to XMPP: #{e}"
+          logger.info "Failed to connect to XMPP: #{e}"
         end
         sleep 1
 
-        log "Authenticating to XMPP"
+        logger.info "Authenticating to XMPP"
         @client.auth(cfg.password)
         sleep 1
 
-        log "Connected to XMPP"
+        logger.info "Connected to XMPP"
 
         @muc = Jabber::MUC::SimpleMUCClient.new(@client)
         @muc.join(cfg.channel + "/" + cfg.nick)
 
-        log "Joined #{cfg.channel}"
+        logger.info "Joined #{cfg.channel}"
       end
     rescue Timeout::Error
-      log "timed out"
+      logger.info "timed out"
       @client = nil
       @muc = nil
     end
 
     @client.on_exception do
-      log "XMPP connection aborted, reconnecting"
+      logger.info "XMPP connection aborted, reconnecting"
       @client = nil
       @muc = nil
       connect
@@ -66,14 +66,14 @@ class XMPPDiff < Oxidized::Hook
           # Maybe connecting failed, so only proceed if we actually joined the MUC
           unless @muc.nil?
             title = "#{ctx.node.name} #{ctx.node.group} #{ctx.node.model.class.name.to_s.downcase}"
-            log "Posting diff as snippet to #{cfg.channel}"
+            logger.info "Posting diff as snippet to #{cfg.channel}"
 
             @muc.say(title + "\n\n" + diff[:patch].lines.to_a[4..-1].join)
           end
         end
       end
     rescue Timeout::Error
-      log "timed out"
+      logger.info "timed out"
     end
   end
 end

data/lib/oxidized/hook.rb

@@ -1,9 +1,13 @@
 module Oxidized
   class HookManager
+    include SemanticLogger::Loggable
+
     class << self
       def from_config(cfg)
         mgr = new
         cfg.hooks.each do |name, h_cfg|
+          raise("Please specify an hook type in the configuration") unless h_cfg.type?
+
           h_cfg.events.each do |event|
             mgr.register event.to_sym, name, h_cfg.type, h_cfg
           end
@@ -12,18 +16,24 @@ module Oxidized
       end
     end
 
-    # HookContext is passed to each hook. It can contain anything related to the
-    # event in question. At least it contains the event name
-    class HookContext < OpenStruct; end
+    # HookContext is passed to each hook. It always carries the event name.
+    # The keyword_init: true argument forces keyword-argument initialization.
+    HookContext = Struct.new(
+      :event, :node, :job, :commitref,
+      :node_raw,   # raw source record: JSON hash, SQL row hash, CSV field array
+      :context,    # self from call site, to access methods/bindings of call site
+      keyword_init: true
+    )
 
     # RegisteredHook is a container for a Hook instance
     RegisteredHook = Struct.new(:name, :hook)
 
-    Events = %i[
+    EVENTS = %i[
       node_success
       node_fail
       post_store
       nodes_done
+      source_node_transform
     ].freeze
     attr_reader :registered_hooks
 
@@ -32,9 +42,9 @@ module Oxidized
     end
 
     def register(event, name, hook_type, cfg)
-      unless Events.include? event
+      unless EVENTS.include? event
         raise ArgumentError,
-              "unknown event #{event}, available: #{Events.join ','}"
+              "unknown event #{event}, available: #{EVENTS.join ','}"
       end
 
       Oxidized.mgr.add_hook(hook_type) || raise("cannot load hook '#{hook_type}', not found")
@@ -47,24 +57,70 @@ module Oxidized
       hook.cfg = cfg
 
       @registered_hooks[event] << RegisteredHook.new(name, hook)
-      Oxidized.logger.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}"
+      logger.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}"
     end
 
+    # --- Transform events ---
+
+    # Runs source_node_transform hooks in sequence, passing the return value of
+    # each hook as node to the next. Returns the final node, or nil
+    # to signal that the node should be excluded.
+    def source_node_transform(node:, node_raw:, context:)
+      ctx = HookContext.new(
+        event:    :source_node_transform,
+        node:     node,
+        node_raw: node_raw,
+        context:  context
+      )
+      @registered_hooks[:source_node_transform].each do |r_hook|
+        ctx.node = r_hook.hook.run_hook(ctx)
+      rescue StandardError => e
+        logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
+                     "(#{e.inspect}) for event :source_node_transform"
+      end
+      ctx.node
+    end
+
+    # --- Fire-and-forget events ---
+
+    def node_success(node:, job: nil)
+      handle(:node_success, node: node, job: job)
+    end
+
+    def node_fail(node:, job: nil)
+      handle(:node_fail, node: node, job: job)
+    end
+
+    def post_store(node:, job: nil, commitref: nil)
+      handle(:post_store, node: node, job: job, commitref: commitref)
+    end
+
+    def nodes_done
+      handle(:nodes_done)
+    end
+
+    private
+
+    # Shared implementation for fire-and-forget events: runs all registered
+    # hooks for the event, ignores return values, logs errors.
     def handle(event, ctx_params = {})
-      ctx = HookContext.new ctx_params
-      ctx.event = event
+      ctx = HookContext.new(event: event, **ctx_params)
 
       @registered_hooks[event].each do |r_hook|
-        r_hook.hook.run_hook ctx
+        r_hook.hook.run_hook(ctx)
       rescue StandardError => e
-        Oxidized.logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
-                              "(#{e.inspect}) for event #{event.inspect}"
+        logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
+                     "(#{e.inspect}) for event #{event.inspect}"
       end
+
+      nil
     end
   end
 
   # Hook abstract base class
   class Hook
+    include SemanticLogger::Loggable
+
     attr_reader :cfg
 
     def cfg=(cfg)
@@ -75,9 +131,5 @@ module Oxidized
     def run_hook(_ctx)
       raise NotImplementedError
     end
-
-    def log(msg, level = :info)
-      Oxidized.logger.send(level, "#{self.class.name}: #{msg}")
-    end
   end
 end

data/lib/oxidized/input/cli.rb

@@ -20,15 +20,16 @@ module Oxidized
       end
 
       def connect_cli
-        Oxidized.logger.debug "lib/oxidized/input/cli.rb: Running post_login commands at #{node.name}"
+        logger.debug "Running post_login commands at #{node.name}"
         @post_login.each do |command, block|
-          Oxidized.logger.debug "lib/oxidized/input/cli.rb: Running post_login command: #{command.inspect}, block: #{block.inspect} at #{node.name}"
+          logger.debug "Running post_login command: #{command.inspect}, " \
+                       "block: #{block.inspect} at #{node.name}"
           block ? block.call : (cmd command)
         end
       end
 
       def disconnect_cli
-        Oxidized.logger.debug "lib/oxidized/input/cli.rb Running pre_logout commands at #{node.name}"
+        logger.debug "Running pre_logout commands at #{node.name}"
         @pre_logout.each { |command, block| block ? block.call : (cmd command, nil) }
       end
 
@@ -52,6 +53,10 @@ module Oxidized
         @password || (@password = regex)
       end
 
+      def newline(newline_str = "\n")
+        @newline || (@newline = newline_str)
+      end
+
       def login
         match_re = [@node.prompt]
         match_re << @username if @username

data/lib/oxidized/input/debugtext.rb

@@ -0,0 +1,40 @@
+module Oxidized
+  class DebugText
+    include SemanticLogger::Loggable
+
+    def initialize(config_debug, node, input_name)
+      return unless config_debug == true ||
+                    (config_debug.is_a?(String) && config_debug.downcase.include?('text'))
+
+      @log = File.open(logfile(node, input_name), 'w')
+    end
+
+    # Separate method to ease unit tests
+    def logfile(node, input_name)
+      timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
+      file = Oxidized::Config::LOG + "/#{node&.ip}-#{input_name}-#{timestamp}.txt"
+      logger.debug "Writing I/O Debugging to #{file}"
+      file
+    end
+
+    def send_data(data)
+      return unless @log
+
+      @log.puts "sent cmd #{data.dump}"
+      @log.flush
+    end
+
+    def receive_data(data)
+      return unless @log
+
+      @log.puts "received #{data.dump}"
+      @log.flush
+    end
+
+    def close
+      return unless @log
+
+      @log.close
+    end
+  end
+end

data/lib/oxidized/input/debugyaml.rb

@@ -0,0 +1,82 @@
+module Oxidized
+  class DebugYAML
+    include SemanticLogger::Loggable
+
+    def initialize(config_debug, node, input_name)
+      return unless config_debug == true ||
+                    (config_debug.is_a?(String) && config_debug.downcase.include?('yaml'))
+
+      @log = File.open(logfile(node, input_name), 'w')
+
+      @partial_line = false
+      @first_line = true
+      @commands_started = false
+
+      @log.puts '---'
+      @log.puts 'init_prompt: |-'
+      @log.flush
+    end
+
+    # Separate method to ease unit tests
+    def logfile(node, input_name)
+      timestamp = Time.now.strftime('%Y%m%d-%H%M%S')
+      file = Oxidized::Config::LOG + "/#{node&.ip}-#{input_name}-#{timestamp}.yaml"
+      logger.debug "Writing YAML Simulation to #{file}"
+      file
+    end
+
+    def send_data(data)
+      return unless @log
+
+      @log.puts
+      @log.puts 'commands:' unless @commands_started
+      @log.puts "  - #{data.dump}: |-"
+      @first_line = true
+      @partial_line = false
+      @commands_started = true
+      @log.flush
+    end
+
+    def receive_data(data)
+      return unless @log
+      return if data.empty?
+
+      lines = data.split("\n", -1)
+
+      lines.each_with_index do |line, idx|
+        is_last = idx == lines.length - 1
+        full_line = is_last ? (data[-1] == "\n") : true
+        # Escape line and strip surrounding double quotes
+        line = line.dump[1..-2]
+        if @first_line
+          # Make sure the leading space of the first line (if present)
+          # is coded with \0x20 or YAML block scalars won't work
+          line.sub!(/^ /, '\x20')
+          @first_line = false
+        end
+
+        # Make sure trailing white spaces are coded with \0x20
+        line.gsub!(/ $/, '\x20')
+
+        output = @partial_line ? line : ('      ' + line)
+        @partial_line = false
+
+        if full_line
+          @log.puts output
+        else
+          @log.write output
+        end
+      end
+
+      @partial_line = data[-1] != "\n"
+
+      @log.flush
+    end
+
+    def close
+      return unless @log
+
+      @log.close
+    end
+  end
+end

data/lib/oxidized/input/exec.rb

@@ -1,17 +1,12 @@
 module Oxidized
-  require "oxidized/input/cli"
-
   class Exec < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-exec", "w") if Oxidized.config.input.debug?
       @node.model.cfg["exec"].each { |cb| instance_exec(&cb) }
     end
 
     def cmd(cmd_str)
-      Oxidized.logger.debug "EXEC: #{cmd_str} @ #{@node.name}"
+      logger.debug "EXEC: #{cmd_str} @ #{@node.name}"
       # I'd really like to do popen3 with separate arguments, but that would
       # require refactoring cmd to take parameters
       %x(#{cmd_str})
@@ -19,10 +14,6 @@ module Oxidized
 
     private
 
-    def disconnect
-      true
-    ensure
-      @log.close if Oxidized.config.input.debug?
-    end
+    def disconnect; end
   end
 end

data/lib/oxidized/input/ftp.rb

@@ -1,24 +1,10 @@
 module Oxidized
   require 'net/ftp'
   require 'timeout'
-  require_relative 'cli'
-
   class FTP < Input
-    RescueFail = {
-      debug: [
-        # Net::SSH::Disconnect,
-      ],
-      warn:  [
-        # RuntimeError,
-        # Net::SSH::AuthenticationFailed,
-      ]
-    }.freeze
-    include Input::CLI
-
-    def connect(node)
+    def connect(node) # rubocop:disable Naming/PredicateMethod
       @node = node
       @node.model.cfg['ftp'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::Log + "/#{@node.ip}-ftp", 'w') if Oxidized.config.input.debug?
       @ftp = Net::FTP.new(@node.ip)
       @ftp.passive = Oxidized.config.input.ftp.passive
       @ftp.login @node.auth[:username], @node.auth[:password]
@@ -30,7 +16,7 @@ module Oxidized
     end
 
     def cmd(file)
-      Oxidized.logger.debug "FTP: #{file} @ #{@node.name}"
+      logger.debug "FTP: #{file} @ #{@node.name}"
       @ftp.getbinaryfile file, nil
     end
 
@@ -47,9 +33,6 @@ module Oxidized
 
     def disconnect
       @ftp.close
-    # rescue Errno::ECONNRESET, IOError
-    ensure
-      @log.close if Oxidized.config.input.debug?
     end
   end
 end