oxidized 0.24.0 → 0.25.0

data/lib/oxidized/model/voss.rb

@@ -14,10 +14,10 @@ class Voss < Oxidized::Model
   # Get sys-info and remove information that changes such has temperature and power
   cmd 'show sys-info' do |cfg|
     cfg.gsub! /(^((.*)SysUpTime(.*))$)/, 'removed SysUpTime'
-    cfg.gsub! /^((.*)Temperature Info \:(.*\r?\n){4})/, 'removed Temperature Info and 3 more lines'
-    cfg.gsub! /(^((.*)AmbientTemperature(.*)\:(.*))$)/, 'removed AmbientTemperature'
-    cfg.gsub! /(^((.*)Temperature(.*)\:(.*))$)/, 'removed Temperature'
-    cfg.gsub! /(^((.*)Total Power Usage(.*)\:(.*))$)/, 'removed Total Power Usage'
+    cfg.gsub! /^((.*)Temperature Info :(.*\r?\n){4})/, 'removed Temperature Info and 3 more lines'
+    cfg.gsub! /(^((.*)AmbientTemperature(.*):(.*))$)/, 'removed AmbientTemperature'
+    cfg.gsub! /(^((.*)Temperature(.*):(.*))$)/, 'removed Temperature'
+    cfg.gsub! /(^((.*)Total Power Usage(.*):(.*))$)/, 'removed Total Power Usage'
     comment "#{cfg}\n"
   end
 

data/lib/oxidized/model/vrp.rb

@@ -11,7 +11,7 @@ class VRP < Oxidized::Model
   end
 
   cmd :all do |cfg|
-    cfg.each_line.to_a[1..-2].join
+    cfg.cut_both
   end
 
   cfg :telnet do

data/lib/oxidized/model/vyatta.rb

@@ -1,7 +1,7 @@
 class Vyatta < Oxidized::Model
   # Brocade Vyatta / VyOS model #
 
-  prompt /\@.*?\:~\$\s/
+  prompt /@.*?:~\$\s/
 
   cmd :all do |cfg|
     cfg = cfg.lines.to_a[1..-2].join

data/lib/oxidized/model/weos.rb

@@ -4,7 +4,7 @@ class WEOS < Oxidized::Model
   prompt /^(\s[\w.@-]+[#>]\s?)$/
 
   cmd :all do |cfg|
-    cfg.each_line.to_a[1..-2].join
+    cfg.cut_both
   end
 
   cmd 'show running-config' do |cfg|

data/lib/oxidized/model/xos.rb

@@ -27,7 +27,7 @@ class XOS < Oxidized::Model
   end
 
   cmd 'show configuration' do |cfg|
-    cfg = cfg.each_line.reject { |line| line.match /^#(\s[\w]+\s)(Configuration generated)/ }.join
+    cfg = cfg.each_line.reject { |line| line.match /^#(\s[\w -]+\s)(Configuration generated)/ }.join
     cfg
   end
 
@@ -41,7 +41,14 @@ class XOS < Oxidized::Model
   end
 
   cfg :telnet, :ssh do
-    post_login 'disable clipaging'
+    post_login do
+      data = cmd 'disable clipaging session'
+      match = data.match /^disable clipaging session\n\*?[\w .-]+(:\d+)? # $/m
+      next if match
+
+      cmd 'disable clipaging'
+    end
+
     pre_logout do
       send "exit\n"
       send "n\n"

data/lib/oxidized/node.rb

@@ -39,6 +39,7 @@ module Oxidized
         # don't try input if model is missing config block, we may need strong config to class_name map
         cfg_name = input.to_s.split('::').last.downcase
         next unless @model.cfg[cfg_name] and not @model.cfg[cfg_name].empty?
+
         @model.input = input = input.new
         if config = run_input(input)
           Oxidized.logger.debug "lib/oxidized/node.rb: #{input.class.name} ran for #{name} successfully"
@@ -74,14 +75,17 @@ module Oxidized
         Oxidized.logger.send(level, '%s raised %s%s with msg "%s"' % [self.ip, err.class, resc, err.message])
         return false
       rescue => err
-        file = Oxidized::Config::Crash + '.' + self.ip.to_s
-        open file, 'w' do |fh|
+        crashdir  = Oxidized.config.crash.directory
+        crashfile = Oxidized.config.crash.hostnames? ? self.name : self.ip.to_s
+        FileUtils.mkdir_p(crashdir) unless File.directory?(crashdir)
+
+        open File.join(crashdir, crashfile), 'w' do |fh|
           fh.puts Time.now.utc
           fh.puts err.message + ' [' + err.class.to_s + ']'
           fh.puts '-' * 50
           fh.puts err.backtrace
         end
-        Oxidized.logger.error '%s raised %s with msg "%s", %s saved' % [self.ip, err.class, err.message, file]
+        Oxidized.logger.error '%s raised %s with msg "%s", %s saved' % [self.ip, err.class, err.message, crashfile]
         return false
       end
     end
@@ -173,32 +177,18 @@ module Oxidized
     end
 
     def resolve_repo opt
-      if is_git? opt
-        remote_repo = Oxidized.config.output.git.repo
-
-        if remote_repo.is_a?(::String)
-          if Oxidized.config.output.git.single_repo? || @group.nil?
-            remote_repo
-          else
-            File.join(File.dirname(remote_repo), @group + '.git')
-          end
-        else
-          remote_repo[@group]
-        end
-      elsif is_gitcrypt? opt
-        remote_repo = Oxidized.config.output.gitcrypt.repo
-
-        if remote_repo.is_a?(::String)
-          if Oxidized.config.output.gitcrypt.single_repo? || @group.nil?
-            remote_repo
-          else
-            File.join(File.dirname(remote_repo), @group + '.git')
-          end
+      type = git_type opt
+      return nil unless type
+
+      remote_repo = Oxidized.config.output.send(type).repo
+      if remote_repo.is_a?(::String)
+        if Oxidized.config.output.send(type).single_repo? || @group.nil?
+          remote_repo
         else
-          remote_repo[@group]
+          File.join(File.dirname(remote_repo), @group + '.git')
         end
       else
-        return
+        remote_repo[@group]
       end
     end
 
@@ -237,12 +227,11 @@ module Oxidized
       value
     end
 
-    def is_git? opt
-      (opt[:output] || Oxidized.config.output.default) == 'git'
-    end
+    def git_type opt
+      type = opt[:output] || Oxidized.config.output.default
+      return nil unless type[0..2] == "git"
 
-    def is_gitcrypt? opt
-      (opt[:output] || Oxidized.config.output.default) == 'gitcrypt'
+      type
     end
   end
 end

data/lib/oxidized/nodes.rb

@@ -16,6 +16,7 @@ module Oxidized
         nodes.each do |node|
           # we want to load specific node(s), not all of them
           next unless node_want? node_want, node
+
           begin
             _node = Node.new node
             new.push _node
@@ -32,6 +33,7 @@ module Oxidized
 
     def node_want? node_want, node
       return true unless node_want
+
       node_want_ip = (IPAddr.new(node_want) rescue false)
       name_is_ip   = (IPAddr.new(node[:name]) rescue false)
       if name_is_ip and node_want_ip == node[:name]
@@ -173,6 +175,7 @@ module Oxidized
         node = find { |n| n.name == node_name }
         output = node.output.new
         raise Oxidized::NotSupported unless output.respond_to? :fetch
+
         yield node, output
       end
     end

data/lib/oxidized/output/git.rb

@@ -167,7 +167,7 @@ module Oxidized
 
       begin
         repo = Rugged::Repository.new repo
-        update_repo repo, file, data, @msg, @user, @email
+        update_repo repo, file, data
       rescue Rugged::OSError, Rugged::RepositoryError => open_error
         begin
           Rugged::Repository.init_at repo, :bare
@@ -178,27 +178,24 @@ module Oxidized
       end
     end
 
-    def update_repo repo, file, data, msg, user, email
+    def update_repo repo, file, data
+      oid_old = repo.blob_at(repo.head.target_id, file) rescue nil
+      return false if oid_old and oid_old.content == data
+
       oid = repo.write data, :blob
       index = repo.index
-      index.read_tree repo.head.target.tree unless repo.empty?
-
-      tree_old = index.write_tree repo
-      index.add :path => file, :oid => oid, :mode => 0100644
-      tree_new = index.write_tree repo
-
-      if tree_old != tree_new
-        repo.config['user.name']  = user
-        repo.config['user.email'] = email
-        @commitref = Rugged::Commit.create(repo,
-                                           :tree       => index.write_tree(repo),
-                                           :message    => msg,
-                                           :parents    => repo.empty? ? [] : [repo.head.target].compact,
-                                           :update_ref => 'HEAD',)
-
-        index.write
-        true
-      end
+      index.add path: file, oid: oid, mode: 0100644
+
+      repo.config['user.name']  = @user
+      repo.config['user.email'] = @email
+      @commitref = Rugged::Commit.create(repo,
+                                         tree:       index.write_tree(repo),
+                                         message:    @msg,
+                                         parents:    repo.empty? ? [] : [repo.head.target].compact,
+                                         update_ref: 'HEAD')
+
+      index.write
+      true
     end
   end
 end

data/lib/oxidized/output/gitcrypt.rb

@@ -4,7 +4,7 @@ module Oxidized
     begin
       require 'git'
     rescue LoadError
-      raise OxidizedError, 'git not found: sudo gem install ruby-git'
+      raise OxidizedError, 'git not found: sudo gem install git'
     end
 
     attr_reader :commitref
@@ -99,6 +99,7 @@ module Oxidized
         else
           File.read path
         end
+
         lock repo
       rescue
         'node not found'

data/lib/oxidized/output/http.rb

@@ -14,28 +14,19 @@ module Oxidized
         raise NoConfig, 'no output http config, edit ~/.config/oxidized/config'
       end
     end
+
     require "net/http"
     require "uri"
     require "json"
+
     def store node, outputs, opt = {}
       @commitref = nil
-      json = JSON.pretty_generate(
-        {
-          'msg' => opt[:msg],
-          'user' => opt[:user],
-          'email' => opt[:email],
-          'group' => opt[:group],
-          'node' => node,
-          'config' => outputs.to_cfg,
-          # actually we need to also iterate outputs, for other types like in gitlab. But most people don't use 'type' functionality.
-        }
-      )
       uri = URI.parse @cfg.url
       http = Net::HTTP.new uri.host, uri.port
       # http.use_ssl = true if uri.scheme = 'https'
       req = Net::HTTP::Post.new(uri.request_uri, initheader = { 'Content-Type' => 'application/json' })
       req.basic_auth @cfg.user, @cfg.password
-      req.body = json
+      req.body = generate_json(node, outputs, opt)
       response = http.request req
 
       case response.code.to_i
@@ -50,5 +41,21 @@ module Oxidized
         Oxidized.logger.info "Configuration http backup for #{node} failed status: #{response.body}"
       end
     end
+
+    private
+
+    def generate_json node, outputs, opt
+      JSON.pretty_generate(
+        {
+          'msg'    => opt[:msg],
+          'user'   => opt[:user],
+          'email'  => opt[:email],
+          'group'  => opt[:group],
+          'node'   => node,
+          'config' => outputs.to_cfg,
+          # actually we need to also iterate outputs, for other types like in gitlab. But most people don't use 'type' functionality.
+        }
+      )
+    end
   end
 end

data/lib/oxidized/source/csv.rb

@@ -20,17 +20,12 @@ module Oxidized
 
     def load _node_want = nil
       nodes = []
-      file = File.expand_path(@cfg.file)
-      file = if @cfg.gpg?
-               crypto = GPGME::Crypto.new password: @cfg.gpg_password
-               file = crypto.decrypt(File.open(file)).to_s
-             else
-               open(file)
-             end
-      file.each_line do |line|
+      open_file.each_line do |line|
         next if line.match(/^\s*#/)
+
         data = line.chomp.split(@cfg.delimiter, -1)
         next if data.empty?
+
         # map node parameters
         keys = {}
         @cfg.map.each do |key, position|
@@ -49,5 +44,17 @@ module Oxidized
       end
       nodes
     end
+
+    private
+
+    def open_file
+      file = File.expand_path(@cfg.file)
+      if @cfg.gpg?
+        crypto = GPGME::Crypto.new password: @cfg.gpg_password
+        crypto.decrypt(File.open(file)).to_s
+      else
+        open(file)
+      end
+    end
   end
 end

data/lib/oxidized/source/http.rb

@@ -12,36 +12,17 @@ module Oxidized
     end
 
     require "net/http"
+    require "net/https"
     require "uri"
     require "json"
 
     def load node_want = nil
       nodes = []
-      uri = URI.parse(@cfg.url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if uri.scheme == 'https'
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
-
-      # map headers
-      headers = {}
-      @cfg.headers.each do |header, value|
-        headers[header] = value
-      end
-
-      req_uri = uri.request_uri
-      if node_want
-        req_uri = "#{req_uri}/#{node_want}"
-      end
-      request = Net::HTTP::Get.new(req_uri, headers)
-      if (@cfg.user? && @cfg.pass?)
-        request.basic_auth(@cfg.user, @cfg.pass)
-      end
-
-      response = http.request(request)
-      data = JSON.parse(response.body)
+      data = JSON.parse(read_http(node_want))
       data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
       data.each do |node|
         next if node.empty?
+
         # map node parameters
         keys = {}
         @cfg.map.each do |key, want_position|
@@ -72,5 +53,28 @@ module Oxidized
       end
       object
     end
+
+    def read_http node_want
+      uri = URI.parse(@cfg.url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
+
+      # map headers
+      headers = {}
+      @cfg.headers.each do |header, value|
+        headers[header] = value
+      end
+
+      req_uri = uri.request_uri
+      if node_want
+        req_uri = "#{req_uri}/#{node_want}"
+      end
+      request = Net::HTTP::Get.new(req_uri, headers)
+      if (@cfg.user? && @cfg.pass?)
+        request.basic_auth(@cfg.user, @cfg.pass)
+      end
+      http.request(request).body
+    end
   end
 end

data/lib/oxidized/string.rb

@@ -4,13 +4,18 @@ module Oxidized
     attr_accessor :type, :cmd, :name
 
     # @return [Oxidized::String] copy of self with last line removed
-    def cut_tail
-      Oxidized::String.new each_line.to_a[0..-2].join
+    def cut_tail lines = 1
+      Oxidized::String.new each_line.to_a[0..-1 - lines].join
     end
 
     # @return [Oxidized::String] copy of self with first line removed
-    def cut_head
-      Oxidized::String.new each_line.to_a[1..-1].join
+    def cut_head lines = 1
+      Oxidized::String.new each_line.to_a[lines..-1].join
+    end
+
+    # @return [Oxidized::String] copy of self with first and last lines removed
+    def cut_both head = 1, tail = 1
+      Oxidized::String.new each_line.to_a[head..-1 - tail].join
     end
 
     # sets @cmd and @name unless @name is already set

data/lib/oxidized/version.rb

@@ -1,6 +1,6 @@
 module Oxidized
-  VERSION = '0.24.0'
-  VERSION_FULL = '0.24.0'
+  VERSION = '0.25.0'
+  VERSION_FULL = '0.25.0'
   def self.version_set
     version_full = %x(git describe --tags).chop rescue ""
     version      = %x(git describe --tags --abbrev=0).chop rescue ""

data/lib/oxidized/worker.rb

@@ -28,8 +28,9 @@ module Oxidized
         # shift nodes and get the next node
         node = @nodes.get
         node.running? ? next : node.running = true
+
         @jobs.push Job.new node
-        Oxidized.logger.debug "lib/oxidized/worker.rb: Added #{node.name} to the job queue"
+        Oxidized.logger.debug "lib/oxidized/worker.rb: Added #{node.group}/#{node.name} to the job queue"
       end
 
       run_done_hook if is_cycle_finished?
@@ -42,49 +43,56 @@ module Oxidized
       node.stats.add job
       @jobs.duration job.time
       node.running = false
-
       if job.status == :success
-        @jobs_done += 1 # needed for :nodes_done hook
-        Oxidized.Hooks.handle :node_success, :node => node,
-                                             :job => job
-        msg = "update #{node.name}"
-        msg += " from #{node.from}" if node.from
-        msg += " with message '#{node.msg}'" if node.msg
-        output = node.output.new
-        if output.store node.name, job.config,
-                        :msg => msg, :email => node.email, :user => node.user, :group => node.group
-          node.modified
-          Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}"
-          Oxidized.Hooks.handle :post_store, :node => node,
-                                             :job => job,
-                                             :commitref => output.commitref
-        end
-        node.reset
+        process_success node, job
       else
-        msg = "#{node.name} status #{job.status}"
-        if node.retry < Oxidized.config.retries
-          node.retry += 1
-          msg += ", retry attempt #{node.retry}"
-          @nodes.next node.name
-        else
-          # Only increment the @jobs_done when we give up retries for a node (or success).
-          # As it would otherwise cause @jobs_done to be incremented with generic retries.
-          # This would cause :nodes_done hook to desync from running at the end of the nodelist and
-          # be fired when the @jobs_done > @nodes.count (could be mid-cycle on the next cycle).
-          @jobs_done += 1
-          msg += ", retries exhausted, giving up"
-          node.retry = 0
-          Oxidized.Hooks.handle :node_fail, :node => node,
-                                            :job => job
-        end
-        Oxidized.logger.warn msg
+        process_failure node, job
       end
     rescue NodeNotFound
-      Oxidized.logger.warn "#{node.name} not found, removed while collecting?"
+      Oxidized.logger.warn "#{node.group}/#{node.name} not found, removed while collecting?"
     end
 
     private
 
+    def process_success node, job
+      @jobs_done += 1 # needed for :nodes_done hook
+      Oxidized.Hooks.handle :node_success, node: node,
+                                           job: job
+      msg = "update #{node.group}/#{node.name}"
+      msg += " from #{node.from}" if node.from
+      msg += " with message '#{node.msg}'" if node.msg
+      output = node.output.new
+      if output.store node.name, job.config,
+                      msg: msg, email: node.email, user: node.user, group: node.group
+        node.modified
+        Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}"
+        Oxidized.Hooks.handle :post_store, node: node,
+                                           job: job,
+                                           commitref: output.commitref
+      end
+      node.reset
+    end
+
+    def process_failure node, job
+      msg = "#{node.group}/#{node.name} status #{job.status}"
+      if node.retry < Oxidized.config.retries
+        node.retry += 1
+        msg += ", retry attempt #{node.retry}"
+        @nodes.next node.name
+      else
+        # Only increment the @jobs_done when we give up retries for a node (or success).
+        # As it would otherwise cause @jobs_done to be incremented with generic retries.
+        # This would cause :nodes_done hook to desync from running at the end of the nodelist and
+        # be fired when the @jobs_done > @nodes.count (could be mid-cycle on the next cycle).
+        @jobs_done += 1
+        msg += ", retries exhausted, giving up"
+        node.retry = 0
+        Oxidized.Hooks.handle :node_fail, :node => node,
+                                          :job => job
+      end
+      Oxidized.logger.warn msg
+    end
+
     def is_cycle_finished?
       if @jobs_done > @nodes.count
         true