oxidized 0.30.1 → 0.31.0

data/lib/oxidized/node.rb

@@ -80,6 +80,8 @@ module Oxidized
         @err_reason = err.message.to_s
         false
       rescue StandardError => e
+        # Send a message in debug mode in case we are not able to create a crashfile
+        Oxidized.logger.send(:debug, '%s raised %s with msg "%s", creating crashfile' % [ip, e.class, e.message])
         crashdir  = Oxidized.config.crash.directory
         crashfile = Oxidized.config.crash.hostnames? ? name : ip.to_s
         FileUtils.mkdir_p(crashdir) unless File.directory?(crashdir)
@@ -198,33 +200,43 @@ module Oxidized
     end
 
     def resolve_key(key, opt, global = nil)
-      # resolve key, first get global, then get group then get node config
+      # resolve key: the priority is as follows: node -> group specific model -> group -> model -> global passed -> global
+      # where node has the highest priority (= if defined, overwrites other values)
       key_sym = key.to_sym
       key_str = key.to_s
-      value   = global
-      Oxidized.logger.debug "node.rb: resolving node key '#{key}', with passed global value of '#{value}' and node value '#{opt[key_sym]}'"
+      model_name = @model.class.name.to_s.downcase
+      Oxidized.logger.debug "node.rb: resolving node key '#{key}', with passed global value of '#{global}' and node value '#{opt[key_sym]}'"
 
-      # global
-      if (not value) && Oxidized.config.has_key?(key_str)
-        value = Oxidized.config[key_str]
-        Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from global"
-      end
+      # Node
+      if opt[key_sym]
+        value = opt[key_sym]
+        Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from node"
 
-      # group
-      if Oxidized.config.groups.has_key?(@group) && Oxidized.config.groups[@group].has_key?(key_str)
+      # Group specific model
+      elsif Oxidized.config.groups.has_key?(@group) && Oxidized.config.groups[@group].models.has_key?(model_name) && Oxidized.config.groups[@group].models[model_name].has_key?(key_str)
+        value = Oxidized.config.groups[@group].models[model_name][key_str]
+        Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from model in group"
+
+      # Group
+      elsif Oxidized.config.groups.has_key?(@group) && Oxidized.config.groups[@group].has_key?(key_str)
         value = Oxidized.config.groups[@group][key_str]
         Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from group"
-      end
 
-      # model
-      if Oxidized.config.models.has_key?(@model.class.name.to_s.downcase) && Oxidized.config.models[@model.class.name.to_s.downcase].has_key?(key_str)
-        value = Oxidized.config.models[@model.class.name.to_s.downcase][key_str]
+      # Model
+      elsif Oxidized.config.models.has_key?(model_name) && Oxidized.config.models[model_name].has_key?(key_str)
+        value = Oxidized.config.models[model_name][key_str]
         Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from model"
-      end
 
-      # node
-      value = opt[key_sym] || value
-      Oxidized.logger.debug "node.rb: returning node key '#{key}' with value '#{value}'"
+      # Global passed
+      elsif global
+        value = global
+        Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from passed global value"
+
+      # Global
+      elsif Oxidized.config.has_key?(key_str)
+        value = Oxidized.config[key_str]
+        Oxidized.logger.debug "node.rb: setting node key '#{key}' to value '#{value}' from global"
+      end
       value
     end
 

data/lib/oxidized/nodes.rb

@@ -61,6 +61,9 @@ module Oxidized
       end
     end
 
+    # Returns the configuration of group/node_name
+    #
+    # #fetch is called by oxidzed-web
     def fetch(node_name, group)
       yield_node_output(node_name) do |node, output|
         output.fetch node, group
@@ -69,7 +72,7 @@ module Oxidized
 
     # @param node [String] name of the node moved into the head of array
     def next(node, opt = {})
-      return unless waiting.find_node_index(node)
+      return if running.find_index(node)
 
       with_lock do
         n = del node
@@ -98,6 +101,9 @@ module Oxidized
       find_index(node) || raise(NodeNotFound, "unable to find '#{node}'")
     end
 
+    # Returns all stored versions of group/node_name
+    #
+    # Called by oxidized-web
     def version(node_name, group)
       yield_node_output(node_name) do |node, output|
         output.version node, group
@@ -116,6 +122,10 @@ module Oxidized
       end
     end
 
+    def find_index(node)
+      index { |e| [e.name, e.ip].include? node }
+    end
+
     private
 
     def initialize(opts = {})
@@ -133,10 +143,6 @@ module Oxidized
       @mutex.synchronize(...)
     end
 
-    def find_index(node)
-      index { |e| [e.name, e.ip].include? node }
-    end
-
     # @param node node which is removed from nodes list
     # @return [Node] deleted node
     def del(node)
@@ -179,6 +185,8 @@ module Oxidized
     def yield_node_output(node_name)
       with_lock do
         node = find { |n| n.name == node_name }
+        raise(NodeNotFound, "unable to find '#{node_name}'") if node.nil?
+
         output = node.output.new
         raise NotSupported unless output.respond_to? :fetch
 

data/lib/oxidized/output/file.rb

@@ -1,55 +1,58 @@
 module Oxidized
-  class OxidizedFile < Output
-    require 'fileutils'
+  module Output
+    # ruby's File class must be accessed with ::File to avoid name conflicts
+    class File < Output
+      require 'fileutils'
 
-    attr_reader :commitref
+      attr_reader :commitref
 
-    def initialize
-      super
-      @cfg = Oxidized.config.output.file
-    end
+      def initialize
+        super
+        @cfg = Oxidized.config.output.file
+      end
 
-    def setup
-      return unless @cfg.empty?
+      def setup
+        return unless @cfg.empty?
 
-      Oxidized.asetus.user.output.file.directory = File.join(Config::ROOT, 'configs')
-      Oxidized.asetus.save :user
-      raise NoConfig, 'no output file config, edit ~/.config/oxidized/config'
-    end
+        Oxidized.asetus.user.output.file.directory = ::File.join(Config::ROOT, 'configs')
+        Oxidized.asetus.save :user
+        raise NoConfig, "no output file config, edit #{Oxidized::Config.configfile}"
+      end
 
-    def store(node, outputs, opt = {})
-      file = File.expand_path @cfg.directory
-      file = File.join File.dirname(file), opt[:group] if opt[:group]
-      FileUtils.mkdir_p file
-      file = File.join file, node
-      File.write(file, outputs.to_cfg)
-      @commitref = file
-    end
+      def store(node, outputs, opt = {})
+        file = ::File.expand_path @cfg.directory
+        file = ::File.join ::File.dirname(file), opt[:group] if opt[:group]
+        FileUtils.mkdir_p file
+        file = ::File.join file, node
+        ::File.write(file, outputs.to_cfg)
+        @commitref = file
+      end
 
-    def fetch(node, group)
-      cfg_dir   = File.expand_path @cfg.directory
-      node_name = node.name
-
-      if group # group is explicitly defined by user
-        cfg_dir = File.join File.dirname(cfg_dir), group
-        File.read File.join(cfg_dir, node_name)
-      elsif File.exist? File.join(cfg_dir, node_name) # node configuration file is stored on base directory
-        File.read File.join(cfg_dir, node_name)
-      else
-        path = Dir.glob(File.join(File.dirname(cfg_dir), '**', node_name)).first # fetch node in all groups
-        File.read path
+      def fetch(node, group)
+        cfg_dir   = ::File.expand_path @cfg.directory
+        node_name = node.name
+
+        if group # group is explicitly defined by user
+          cfg_dir = ::File.join ::File.dirname(cfg_dir), group
+          ::File.read ::File.join(cfg_dir, node_name)
+        elsif ::File.exist? ::File.join(cfg_dir, node_name) # node configuration file is stored on base directory
+          ::File.read ::File.join(cfg_dir, node_name)
+        else
+          path = Dir.glob(::File.join(::File.dirname(cfg_dir), '**', node_name)).first # fetch node in all groups
+          ::File.read path
+        end
+      rescue Errno::ENOENT
+        nil
       end
-    rescue Errno::ENOENT
-      nil
-    end
 
-    def version(_node, _group)
-      # not supported
-      []
-    end
+      def version(_node, _group)
+        # not supported
+        []
+      end
 
-    def get_version(_node, _group, _oid)
-      'not supported'
+      def get_version(_node, _group, _oid)
+        'not supported'
+      end
     end
   end
 end

data/lib/oxidized/output/git.rb

@@ -1,195 +1,220 @@
 module Oxidized
-  class Git < Output
-    using Refinements
-
-    class GitError < OxidizedError; end
-    begin
-      require 'rugged'
-    rescue LoadError
-      raise OxidizedError, 'rugged not found: sudo gem install rugged'
-    end
+  module Output
+    class Git < Output
+      using Refinements
 
-    attr_reader :commitref
+      class GitError < OxidizedError; end
+      begin
+        require 'rugged'
+      rescue LoadError
+        raise OxidizedError, 'rugged not found: sudo gem install rugged'
+      end
 
-    def initialize
-      super
-      @cfg = Oxidized.config.output.git
-    end
+      attr_reader :commitref
 
-    def setup
-      if @cfg.empty?
-        Oxidized.asetus.user.output.git.user  = 'Oxidized'
-        Oxidized.asetus.user.output.git.email = 'o@example.com'
-        Oxidized.asetus.user.output.git.repo = File.join(Config::ROOT, 'oxidized.git')
-        Oxidized.asetus.save :user
-        raise NoConfig, 'no output git config, edit ~/.config/oxidized/config'
+      def initialize
+        super
+        @cfg = Oxidized.config.output.git
       end
 
-      if @cfg.repo.respond_to?(:each)
-        @cfg.repo.each do |group, repo|
-          @cfg.repo["#{group}="] = File.expand_path repo
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.output.git.user  = 'Oxidized'
+          Oxidized.asetus.user.output.git.email = 'o@example.com'
+          Oxidized.asetus.user.output.git.repo = File.join(Config::ROOT, 'oxidized.git')
+          Oxidized.asetus.save :user
+          raise NoConfig, "no output git config, edit #{Oxidized::Config.configfile}"
         end
-      else
-        @cfg.repo = File.expand_path @cfg.repo
-      end
-    end
 
-    def store(file, outputs, opt = {})
-      @msg   = opt[:msg]
-      @user  = opt[:user]  || @cfg.user
-      @email = opt[:email] || @cfg.email
-      @opt   = opt
-      @commitref = nil
-      repo = @cfg.repo
-
-      outputs.types.each do |type|
-        type_cfg = ''
-        type_repo = File.join(File.dirname(repo), type + '.git')
-        outputs.type(type).each do |output|
-          (type_cfg << output; next) unless output.name # rubocop:disable Style/Semicolon
-          type_file = file + '--' + output.name
-          if @cfg.type_as_directory?
-            type_file = type + '/' + type_file
-            type_repo = repo
+        if @cfg.repo.respond_to?(:each)
+          @cfg.repo.each do |group, repo|
+            @cfg.repo["#{group}="] = File.expand_path repo
           end
-          update type_repo, type_file, output
+        else
+          @cfg.repo = File.expand_path @cfg.repo
         end
-        update type_repo, file, type_cfg
       end
 
-      update repo, file, outputs.to_cfg
-    end
+      def store(file, outputs, opt = {})
+        @msg   = opt[:msg]
+        @user  = opt[:user]  || @cfg.user
+        @email = opt[:email] || @cfg.email
+        @opt   = opt
+        @commitref = nil
+        repo = @cfg.repo
+
+        outputs.types.each do |type|
+          type_cfg = ''
+          type_repo = File.join(File.dirname(repo), type + '.git')
+          outputs.type(type).each do |output|
+            (type_cfg << output; next) unless output.name # rubocop:disable Style/Semicolon
+            type_file = file + '--' + output.name
+            if @cfg.type_as_directory?
+              type_file = type + '/' + type_file
+              type_repo = repo
+            end
+            update type_repo, type_file, output
+          end
+          update type_repo, file, type_cfg
+        end
 
-    def fetch(node, group)
-      repo, path = yield_repo_and_path(node, group)
-      repo = Rugged::Repository.new repo
-      index = repo.index
-      index.read_tree repo.head.target.tree unless repo.empty?
-      repo.read(index.get(path)[:oid]).data
-    rescue StandardError
-      'node not found'
-    end
+        update repo, file, outputs.to_cfg
+      end
 
-    # give a hash of all oid revision for the given node, and the date of the commit
-    def version(node, group)
-      repo, path = yield_repo_and_path(node, group)
-
-      repo = Rugged::Repository.new repo
-      walker = Rugged::Walker.new(repo)
-      walker.sorting(Rugged::SORT_DATE)
-      walker.push(repo.head.target.oid)
-      i = -1
-      tab = []
-      walker.each do |commit|
-        # Diabled rubocop because the suggested .empty? does not work here.
-        next if commit.diff(paths: [path]).size.zero? # rubocop:disable Style/ZeroLengthPredicate
-
-        hash = {}
-        hash[:date] = commit.time.to_s
-        hash[:oid] = commit.oid
-        hash[:author] = commit.author
-        hash[:message] = commit.message
-        tab[i += 1] = hash
+      # Returns the configuration of group/node_name
+      #
+      # #fetch is called by Nodes#fetch
+      # Nodes#fetch creates a new Output object each time, so we cannot
+      # store the repo index in memory. But as we keep the repo index up
+      # to date on disk in #update_repo, we can read it from disk instead of
+      # rebuilding it each time.
+      def fetch(node, group)
+        repo, path = yield_repo_and_path(node, group)
+        repo = Rugged::Repository.new repo
+        # Read the index from disk
+        index = repo.index
+
+        repo.read(index.get(path)[:oid]).data
+      rescue StandardError
+        'node not found'
       end
-      walker.reset
-      tab
-    rescue StandardError
-      'node not found'
-    end
 
-    # give the blob of a specific revision
-    def get_version(node, group, oid)
-      repo, path = yield_repo_and_path(node, group)
-      repo = Rugged::Repository.new repo
-      repo.blob_at(oid, path).content
-    rescue StandardError
-      'version not found'
-    end
+      # give a hash of all oid revision for the given node, and the date of the commit
+      def version(node, group)
+        repo, path = yield_repo_and_path(node, group)
 
-    # give a hash with the patch of a diff between 2 revision and the stats (added and deleted lines)
-    def get_diff(node, group, oid1, oid2)
-      diff_commits = nil
-      repo, = yield_repo_and_path(node, group)
-      repo = Rugged::Repository.new repo
-      commit = repo.lookup(oid1)
-
-      if oid2
-        commit_old = repo.lookup(oid2)
-        diff = repo.diff(commit_old, commit)
-        diff.each do |patch|
-          if /#{node.name}\s+/ =~ patch.to_s.lines.first
-            diff_commits = { patch: patch.to_s, stat: patch.stat }
-            break
-          end
+        repo = Rugged::Repository.new repo
+        walker = Rugged::Walker.new(repo)
+        walker.sorting(Rugged::SORT_DATE)
+        walker.push(repo.head.target.oid)
+        i = -1
+        tab = []
+        walker.each do |commit|
+          # Diabled rubocop because the suggested .empty? does not work here.
+          next if commit.diff(paths: [path]).size.zero? # rubocop:disable Style/ZeroLengthPredicate
+
+          hash = {}
+          hash[:date] = commit.time.to_s
+          hash[:oid] = commit.oid
+          hash[:author] = commit.author
+          hash[:message] = commit.message
+          tab[i += 1] = hash
         end
-      else
-        stat = commit.parents[0].diff(commit).stat
-        stat = [stat[1], stat[2]]
-        patch = commit.parents[0].diff(commit).patch
-        diff_commits = { patch: patch, stat: stat }
+        walker.reset
+        tab
+      rescue StandardError
+        'node not found'
       end
 
-      diff_commits
-    rescue StandardError
-      'no diffs'
-    end
+      # give the blob of a specific revision
+      def get_version(node, group, oid)
+        repo, path = yield_repo_and_path(node, group)
+        repo = Rugged::Repository.new repo
+        repo.blob_at(oid, path).content
+      rescue StandardError
+        'version not found'
+      end
 
-    private
+      # give a hash with the patch of a diff between 2 revision and the stats (added and deleted lines)
+      def get_diff(node, group, oid1, oid2)
+        diff_commits = nil
+        repo, = yield_repo_and_path(node, group)
+        repo = Rugged::Repository.new repo
+        commit = repo.lookup(oid1)
+
+        if oid2
+          commit_old = repo.lookup(oid2)
+          diff = repo.diff(commit_old, commit)
+          diff.each do |patch|
+            if /#{node.name}\s+/ =~ patch.to_s.lines.first
+              diff_commits = { patch: patch.to_s, stat: patch.stat }
+              break
+            end
+          end
+        else
+          stat = commit.parents[0].diff(commit).stat
+          stat = [stat[1], stat[2]]
+          patch = commit.parents[0].diff(commit).patch
+          diff_commits = { patch: patch, stat: stat }
+        end
 
-    def yield_repo_and_path(node, group)
-      repo, path = node.repo, node.name
+        diff_commits
+      rescue StandardError
+        'no diffs'
+      end
 
-      path = "#{group}/#{node.name}" if group && @cfg.single_repo?
+      private
 
-      [repo, path]
-    end
+      def yield_repo_and_path(node, group)
+        repo, path = node.repo, node.name
 
-    def update(repo, file, data)
-      return if data.empty?
+        path = "#{group}/#{node.name}" if group && !group.empty? && @cfg.single_repo?
 
-      if @opt[:group]
-        if @cfg.single_repo?
-          file = File.join @opt[:group], file
-        else
-          repo = if repo.is_a?(::String)
-                   File.join File.dirname(repo), @opt[:group] + '.git'
-                 else
-                   repo[@opt[:group]]
-                 end
-        end
+        [repo, path]
       end
 
-      begin
-        repo = Rugged::Repository.new repo
-        update_repo repo, file, data
-      rescue Rugged::OSError, Rugged::RepositoryError => e
+      def update(repo, file, data)
+        return if data.empty?
+
+        if @opt[:group]
+          if @cfg.single_repo?
+            file = File.join @opt[:group], file
+          else
+            repo = if repo.is_a?(::String)
+                     File.join File.dirname(repo), @opt[:group] + '.git'
+                   else
+                     repo[@opt[:group]]
+                   end
+          end
+        end
+
         begin
-          Rugged::Repository.init_at repo, :bare
-        rescue StandardError => create_error
-          raise GitError, "first '#{e.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo"
+          repo = Rugged::Repository.new repo
+          update_repo repo, file, data
+        rescue Rugged::OSError, Rugged::RepositoryError => e
+          begin
+            Rugged::Repository.init_at repo, :bare
+          rescue StandardError => create_error
+            raise GitError, "first '#{e.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo"
+          end
+          retry
         end
-        retry
       end
-    end
-
-    def update_repo(repo, file, data)
-      oid_old = repo.blob_at(repo.head.target_id, file) rescue nil
-      return false if oid_old && (oid_old.content.b == data.b)
-
-      oid = repo.write data, :blob
-      index = repo.index
-      index.add path: file, oid: oid, mode: 0o100644
 
-      repo.config['user.name']  = @user
-      repo.config['user.email'] = @email
-      @commitref = Rugged::Commit.create(repo,
-                                         tree:       index.write_tree(repo),
-                                         message:    @msg,
-                                         parents:    repo.empty? ? [] : [repo.head.target].compact,
-                                         update_ref: 'HEAD')
-
-      index.write
-      true
+      # Uploads data into file in the repo
+      #
+      # @param [String] file: the file to save the configuration to
+      # @param [String] data: the configuration to save
+      # @param [Rugged::Repository] repo: the git repository to use
+      #
+      # If Oxidized.config.output.git.single_repo = false (which is the default),
+      # there will one repository for each group.
+      #
+      # update_repo caches the index on disk. An index is usually used in a
+      # working directory and not in a bare repository, which confuses users.
+      # The alternative would be to rebuild the index each time, which a little
+      # time consuming. Caching the index in memory is difficult because a new
+      # Output object is created each time #store is called.
+      def update_repo(repo, file, data)
+        oid_old = repo.blob_at(repo.head.target_id, file) rescue nil
+        return false if oid_old && (oid_old.content.b == data.b)
+
+        oid = repo.write data, :blob
+        # Read the index from disk
+        index = repo.index
+        index.add path: file, oid: oid, mode: 0o100644
+
+        repo.config['user.name']  = @user
+        repo.config['user.email'] = @email
+        @commitref = Rugged::Commit.create(repo,
+                                           tree:       index.write_tree(repo),
+                                           message:    @msg,
+                                           parents:    repo.empty? ? [] : [repo.head.target].compact,
+                                           update_ref: 'HEAD')
+
+        index.write
+        true
+      end
     end
   end
 end