oxidized 0.20.0 → 0.28.0

data/lib/oxidized/source/csv.rb

@@ -1,54 +1,60 @@
 module Oxidized
-class CSV < Source
-  def initialize
-    @cfg = Oxidized.config.source.csv
-    super
-  end
-
-  def setup
-    if @cfg.empty?
-      Oxidized.asetus.user.source.csv.file      = File.join(Config::Root, 'router.db')
-      Oxidized.asetus.user.source.csv.delimiter = /:/
-      Oxidized.asetus.user.source.csv.map.name  = 0
-      Oxidized.asetus.user.source.csv.map.model = 1
-      Oxidized.asetus.user.source.csv.gpg       = false
-      Oxidized.asetus.save :user
-      raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
+  class CSV < Source
+    def initialize
+      @cfg = Oxidized.config.source.csv
+      super
     end
-    require 'gpgme' if @cfg.gpg?
-  end
 
-  def load
-    nodes = []
-    file = File.expand_path(@cfg.file)
-    file = if @cfg.gpg?
-      crypto = GPGME::Crypto.new password: @cfg.gpg_password
-      crypto.decrypt(file).to_s
-    else
-      open(file)
-    end  
-    file.each_line do |line|
-      next if line.match(/^\s*#/)
-      data  = line.chomp.split(@cfg.delimiter, -1)
-      next if data.empty?
-      # map node parameters
-      keys = {}
-      @cfg.map.each do |key, position|
-        keys[key.to_sym] = node_var_interpolate data[position]
+    def setup
+      if @cfg.empty?
+        Oxidized.asetus.user.source.csv.file      = File.join(Config::Root, 'router.db')
+        Oxidized.asetus.user.source.csv.delimiter = /:/
+        Oxidized.asetus.user.source.csv.map.name  = 0
+        Oxidized.asetus.user.source.csv.map.model = 1
+        Oxidized.asetus.user.source.csv.gpg       = false
+        Oxidized.asetus.save :user
+        raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
       end
-      keys[:model] = map_model keys[:model] if keys.key? :model
+      require 'gpgme' if @cfg.gpg?
+    end
+
+    def load(_node_want = nil)
+      nodes = []
+      open_file.each_line do |line|
+        next if line =~ /^\s*#/
+
+        data = line.chomp.split(@cfg.delimiter, -1)
+        next if data.empty?
+
+        # map node parameters
+        keys = {}
+        @cfg.map.each do |key, position|
+          keys[key.to_sym] = node_var_interpolate data[position]
+        end
+        keys[:model] = map_model keys[:model] if keys.has_key? :model
 
-      # map node specific vars
-      vars = {}
-      @cfg.vars_map.each do |key, position|
-        vars[key.to_sym] = node_var_interpolate data[position]
+        # map node specific vars
+        vars = {}
+        @cfg.vars_map.each do |key, position|
+          vars[key.to_sym] = node_var_interpolate data[position]
+        end
+        keys[:vars] = vars unless vars.empty?
+
+        nodes << keys
       end
-      keys[:vars] = vars unless vars.empty?
+      nodes
+    end
+
+    private
 
-      nodes << keys
+    def open_file
+      file = File.expand_path(@cfg.file)
+      if @cfg.gpg?
+        crypto = GPGME::Crypto.new password: @cfg.gpg_password
+        crypto.decrypt(File.open(file)).to_s
+      else
+        File.open(file)
+      end
     end
-    nodes
   end
-
-end
 end

data/lib/oxidized/source/http.rb

@@ -1,77 +1,77 @@
 module Oxidized
-class HTTP < Source
-  def initialize
-    @cfg = Oxidized.config.source.http
-    super
-  end
+  class HTTP < Source
+    def initialize
+      @cfg = Oxidized.config.source.http
+      super
+    end
+
+    def setup
+      return unless @cfg.url.empty?
 
-  def setup
-    if @cfg.url.empty?
       raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config'
     end
-  end
 
-  require "net/http"
-  require "uri"
-  require "json"
+    require "net/http"
+    require "net/https"
+    require "uri"
+    require "json"
 
-  def load
-    nodes = []
-    uri = URI.parse(@cfg.url)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.use_ssl = true if uri.scheme == 'https'
-    http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
+    def load(node_want = nil)
+      nodes = []
+      data = JSON.parse(read_http(node_want))
+      data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+      data.each do |node|
+        next if node.empty?
 
-    # map headers
-    headers = {}
-    @cfg.headers.each do |header, value|
-      headers[header] = value
-    end
+        # map node parameters
+        keys = {}
+        @cfg.map.each do |key, want_position|
+          keys[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+        end
+        keys[:model] = map_model keys[:model] if keys.has_key? :model
 
-    request = Net::HTTP::Get.new(uri.request_uri, headers)
-    if (@cfg.user? && @cfg.pass?)
-        request.basic_auth(@cfg.user,@cfg.pass)
-    end
+        # map node specific vars
+        vars = {}
+        @cfg.vars_map.each do |key, want_position|
+          vars[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+        end
+        keys[:vars] = vars unless vars.empty?
 
-    response = http.request(request)
-    data = JSON.parse(response.body)
-    data.each do |node|
-      next if node.empty?
-      # map node parameters
-      keys = {}
-      @cfg.map.each do |key, want_position|
-        want_positions = want_position.split('.')
-        keys[key.to_sym] = node_var_interpolate node.dig(*want_positions)
+        nodes << keys
       end
-      keys[:model] = map_model keys[:model] if keys.key? :model
+      nodes
+    end
 
-      # map node specific vars
-      vars = {}
-      @cfg.vars_map.each do |key, want_position|
-        want_positions = want_position.split('.')
-        vars[key.to_sym] = node_var_interpolate node.dig(*want_positions)
-      end
-      keys[:vars] = vars unless vars.empty?
+    private
 
-      nodes << keys
+    def string_navigate(object, wants)
+      wants = wants.split(".").map do |want|
+        head, match, _tail = want.partition(/\[\d+\]/)
+        match.empty? ? head : [head, match[1..-2].to_i]
+      end
+      wants.flatten.each do |want|
+        object = object[want] if object.respond_to? :each
+      end
+      object
     end
-    nodes
-  end
 
-end
-end
+    def read_http(node_want)
+      uri = URI.parse(@cfg.url)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == 'https'
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
 
-if RUBY_VERSION < '2.3'
-  class Hash
-    def dig(key, *rest)
-      value = self[key]
-      if value.nil? || rest.empty?
-        value
-      elsif value.respond_to?(:dig)
-        value.dig(*rest)
-      else # foo.bar.baz (bar exist but is not hash)
-        return nil
+      # map headers
+      headers = {}
+      @cfg.headers.each do |header, value|
+        headers[header] = value
       end
+
+      req_uri = uri.request_uri
+      req_uri = "#{req_uri}/#{node_want}" if node_want
+      request = Net::HTTP::Get.new(req_uri, headers)
+      request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
+      http.request(request).body
     end
   end
 end

data/lib/oxidized/source/source.rb

@@ -3,21 +3,20 @@ module Oxidized
     class NoConfig < OxidizedError; end
 
     def initialize
-      @map = (Oxidized.config.model_map or {})
+      @map = (Oxidized.config.model_map || {})
     end
 
-    def map_model model
+    def map_model(model)
       @map.has_key?(model) ? @map[model] : model
     end
 
-    def node_var_interpolate var
-       case var
-       when "nil"   then nil
-       when "false" then false
-       when "true"  then true
-       else var
-       end
+    def node_var_interpolate(var)
+      case var
+      when "nil"   then nil
+      when "false" then false
+      when "true"  then true
+      else var
+      end
     end
-
   end
 end

data/lib/oxidized/source/sql.rb

@@ -1,13 +1,14 @@
 module Oxidized
-class SQL < Source
-  begin
-    require 'sequel'
-  rescue LoadError
-    raise OxidizedError, 'sequel not found: sudo gem install sequel'
-  end
+  class SQL < Source
+    begin
+      require 'sequel'
+    rescue LoadError
+      raise OxidizedError, 'sequel not found: sudo gem install sequel'
+    end
+
+    def setup
+      return unless @cfg.empty?
 
-  def setup
-    if @cfg.empty?
       Oxidized.asetus.user.source.sql.adapter   = 'sqlite'
       Oxidized.asetus.user.source.sql.database  = File.join(Config::Root, 'sqlite.db')
       Oxidized.asetus.user.source.sql.table     = 'devices'
@@ -16,48 +17,49 @@ class SQL < Source
       Oxidized.asetus.save :user
       raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
     end
-  end
 
-  def load
-    nodes = []
-    db = connect
-    query = db[@cfg.table.to_sym]
-    query = query.with_sql(@cfg.query) if @cfg.query?
-    query.each do |node|
-      # map node parameters
-      keys = {}
-      @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
-      keys[:model] = map_model keys[:model] if keys.key? :model
-
-      # map node specific vars
-      vars = {}
-      @cfg.vars_map.each do |key, sql_column|
-        vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym]
-      end
-      keys[:vars] = vars unless vars.empty?
+    def load(node_want = nil)
+      nodes = []
+      db = connect
+      query = db[@cfg.table.to_sym]
+      query = query.with_sql(@cfg.query) if @cfg.query?
+
+      query = query.where(@cfg.map.name.to_sym => node_want) if node_want
+
+      query.each do |node|
+        # map node parameters
+        keys = {}
+        @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
+        keys[:model] = map_model keys[:model] if keys.has_key? :model
+
+        # map node specific vars
+        vars = {}
+        @cfg.vars_map.each do |key, sql_column|
+          vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym]
+        end
+        keys[:vars] = vars unless vars.empty?
 
-      nodes << keys
+        nodes << keys
+      end
+      db.disconnect
+      nodes
     end
-    db.disconnect
-    nodes
-  end
 
-  private
+    private
 
-  def initialize
-    super
-    @cfg = Oxidized.config.source.sql
-  end
+    def initialize
+      super
+      @cfg = Oxidized.config.source.sql
+    end
 
-  def connect
-    Sequel.connect(:adapter  => @cfg.adapter,
-                   :host     => @cfg.host?,
-                   :user     => @cfg.user?,
-                   :password => @cfg.password?,
-                   :database => @cfg.database)
-  rescue Sequel::AdapterNotFound => error
-    raise OxidizedError, "SQL adapter gem not installed: " + error.message
+    def connect
+      Sequel.connect(adapter:  @cfg.adapter,
+                     host:     @cfg.host?,
+                     user:     @cfg.user?,
+                     password: @cfg.password?,
+                     database: @cfg.database)
+    rescue Sequel::AdapterNotFound => error
+      raise OxidizedError, "SQL adapter gem not installed: " + error.message
+    end
   end
-
-end
 end

data/lib/oxidized/string.rb

@@ -4,29 +4,33 @@ module Oxidized
     attr_accessor :type, :cmd, :name
 
     # @return [Oxidized::String] copy of self with last line removed
-    def cut_tail
-      Oxidized::String.new each_line.to_a[0..-2].join
+    def cut_tail(lines = 1)
+      Oxidized::String.new each_line.to_a[0..-1 - lines].join
     end
 
     # @return [Oxidized::String] copy of self with first line removed
-    def cut_head
-      Oxidized::String.new each_line.to_a[1..-1].join
+    def cut_head(lines = 1)
+      Oxidized::String.new each_line.to_a[lines..-1].join
+    end
+
+    # @return [Oxidized::String] copy of self with first and last lines removed
+    def cut_both(head = 1, tail = 1)
+      Oxidized::String.new each_line.to_a[head..-1 - tail].join
     end
 
     # sets @cmd and @name unless @name is already set
-    def set_cmd command
-      @cmd  = command
-      @name ||= @cmd.strip.gsub(/\s+/, '_')
+    def set_cmd(command)
+      @cmd = command
+      @name ||= @cmd.to_s.strip.gsub(/\s+/, '_') # what to do when command is proc? #to_s seems ghetto
     end
 
-    def initialize str=''
+    def initialize(str = '')
       super
-      if str.class == Oxidized::String
-        @cmd  = str.cmd
-        @name = str.name
-        @type = str.type
-      end
-    end
+      return unless str.class == Oxidized::String
 
+      @cmd  = str.cmd
+      @name = str.name
+      @type = str.type
+    end
   end
 end

data/lib/oxidized/version.rb

@@ -1,3 +1,19 @@
 module Oxidized
-  VERSION = '0.20.0'
+  VERSION = '0.28.0'.freeze
+  VERSION_FULL = '0.28.0'.freeze
+  def self.version_set
+    version_full = %x(git describe --tags).chop rescue ""
+    version      = %x(git describe --tags --abbrev=0).chop rescue ""
+
+    return false unless [version, version_full].none?(&:empty?)
+
+    Oxidized.send(:remove_const, :VERSION)
+    Oxidized.send(:remove_const, :VERSION_FULL)
+    const_set(:VERSION, version)
+    const_set(:VERSION_FULL, version_full)
+    file = File.readlines(__FILE__)
+    file[1] = "  VERSION = '%s'.freeze\n" % VERSION
+    file[2] = "  VERSION_FULL = '%s'.freeze\n" % VERSION_FULL
+    File.write(__FILE__, file.join)
+  end
 end

data/lib/oxidized/worker.rb

@@ -2,7 +2,8 @@ module Oxidized
   require 'oxidized/job'
   require 'oxidized/jobs'
   class Worker
-    def initialize nodes
+    def initialize(nodes)
+      @jobs_done  = 0
       @nodes      = nodes
       @jobs       = Jobs.new(Oxidized.config.threads, Oxidized.config.interval, @nodes)
       @nodes.jobs = @jobs
@@ -11,65 +12,103 @@ module Oxidized
 
     def work
       ended = []
-      @jobs.delete_if { |job| ended << job if not job.alive? }
+      @jobs.delete_if { |job| ended << job unless job.alive? }
       ended.each      { |job| process job }
       @jobs.work
+
       while @jobs.size < @jobs.want
-        Oxidized.logger.debug "lib/oxidized/worker.rb: Jobs #{@jobs.size}, Want: #{@jobs.want}"
+        Oxidized.logger.debug "lib/oxidized/worker.rb: Jobs running: #{@jobs.size} of #{@jobs.want} - ended: #{@jobs_done} of #{@nodes.size}"
         # ask for next node in queue non destructive way
         nextnode = @nodes.first
         unless nextnode.last.nil?
           # Set unobtainable value for 'last' if interval checking is disabled
-          last = Oxidized.config.interval == 0 ? Time.now.utc + 10 : nextnode.last.end
+          last = Oxidized.config.interval.zero? ? Time.now.utc + 10 : nextnode.last.end
           break if last + Oxidized.config.interval > Time.now.utc
         end
         # shift nodes and get the next node
         node = @nodes.get
         node.running? ? next : node.running = true
+
         @jobs.push Job.new node
-        Oxidized.logger.debug "lib/oxidized/worker.rb: Added #{node.name} to the job queue"
+        Oxidized.logger.debug "lib/oxidized/worker.rb: Added #{node.group}/#{node.name} to the job queue"
       end
+
+      run_done_hook if cycle_finished?
       Oxidized.logger.debug("lib/oxidized/worker.rb: #{@jobs.size} jobs running in parallel") unless @jobs.empty?
     end
 
-    def process job
+    def process(job)
       node = job.node
       node.last = job
       node.stats.add job
       @jobs.duration job.time
       node.running = false
       if job.status == :success
-        Oxidized.Hooks.handle :node_success, :node => node,
-                                             :job => job
-        msg = "update #{node.name}"
-        msg += " from #{node.from}" if node.from
-        msg += " with message '#{node.msg}'" if node.msg
-        output = node.output.new
-        if output.store node.name, job.config,
-                              :msg => msg, :user => node.user, :group => node.group
-          Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}"
-          Oxidized.Hooks.handle :post_store, :node => node,
-                                             :job => job,
-                                             :commitref => output.commitref
-        end
-        node.reset
+        process_success node, job
       else
-        msg = "#{node.name} status #{job.status}"
-        if node.retry < Oxidized.config.retries
-          node.retry += 1
-          msg += ", retry attempt #{node.retry}"
-          @nodes.next node.name
-        else
-          msg += ", retries exhausted, giving up"
-          node.retry = 0
-          Oxidized.Hooks.handle :node_fail, :node => node,
-                                            :job => job
-        end
-        Oxidized.logger.warn msg
+        process_failure node, job
       end
     rescue NodeNotFound
-      Oxidized.logger.warn "#{node.name} not found, removed while collecting?"
+      Oxidized.logger.warn "#{node.group}/#{node.name} not found, removed while collecting?"
+    end
+
+    private
+
+    def process_success(node, job)
+      @jobs_done += 1 # needed for :nodes_done hook
+      Oxidized.Hooks.handle :node_success, node: node,
+                                           job:  job
+      msg = "update #{node.group}/#{node.name}"
+      msg += " from #{node.from}" if node.from
+      msg += " with message '#{node.msg}'" if node.msg
+      output = node.output.new
+      if output.store node.name, job.config,
+                      msg: msg, email: node.email, user: node.user, group: node.group
+        node.modified
+        Oxidized.logger.info "Configuration updated for #{node.group}/#{node.name}"
+        Oxidized.Hooks.handle :post_store, node:      node,
+                                           job:       job,
+                                           commitref: output.commitref
+      end
+      node.reset
     end
 
+    def process_failure(node, job)
+      msg = "#{node.group}/#{node.name} status #{job.status}"
+      if node.retry < Oxidized.config.retries
+        node.retry += 1
+        msg += ", retry attempt #{node.retry}"
+        @nodes.next node.name
+      else
+        # Only increment the @jobs_done when we give up retries for a node (or success).
+        # As it would otherwise cause @jobs_done to be incremented with generic retries.
+        # This would cause :nodes_done hook to desync from running at the end of the nodelist and
+        # be fired when the @jobs_done > @nodes.count (could be mid-cycle on the next cycle).
+        @jobs_done += 1
+        msg += ", retries exhausted, giving up"
+        node.retry = 0
+        Oxidized.Hooks.handle :node_fail, node: node,
+                                          job:  job
+      end
+      Oxidized.logger.warn msg
+    end
+
+    def cycle_finished?
+      if @jobs_done > @nodes.count
+        true
+      else
+        @jobs_done.positive? && (@jobs_done % @nodes.count).zero?
+      end
+    end
+
+    def run_done_hook
+      Oxidized.logger.debug "lib/oxidized/worker.rb: Running :nodes_done hook"
+      Oxidized.Hooks.handle :nodes_done
+    rescue StandardError => e
+      # swallow the hook erros and continue as normal
+      Oxidized.logger.error "lib/oxidized/worker.rb: #{e.message}"
+    ensure
+      @jobs_done = 0
+    end
   end
 end

data/oxidized.gemspec

@@ -1,37 +1,38 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'oxidized/version'
 
 Gem::Specification.new do |s|
   s.name              = 'oxidized'
   s.version           = Oxidized::VERSION
-  s.licenses          = %w( Apache-2.0 )
+  s.licenses          = %w[Apache-2.0]
   s.platform          = Gem::Platform::RUBY
-  s.authors           = [ 'Saku Ytti', 'Samer Abdel-Hafez', 'Anton Aksola' ]
-  s.email             = %w( saku@ytti.fi sam@arahant.net aakso@iki.fi)
+  s.authors           = ['Saku Ytti', 'Samer Abdel-Hafez', 'Anton Aksola']
+  s.email             = %w[saku@ytti.fi sam@arahant.net aakso@iki.fi]
   s.homepage          = 'http://github.com/ytti/oxidized'
   s.summary           = 'feeble attempt at rancid'
   s.description       = 'software to fetch configuration from network devices and store them'
   s.rubyforge_project = s.name
-  s.files             = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  s.executables       = %w( oxidized )
+  s.files             = %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.executables       = %w[oxidized]
   s.require_path      = 'lib'
 
-  s.required_ruby_version =           '>= 2.0.0'
+  s.required_ruby_version =           '>= 2.3'
   s.add_runtime_dependency 'asetus',  '~> 0.1'
-  s.add_runtime_dependency 'slop',    '~> 3.5'
-  s.add_runtime_dependency 'net-ssh', '~> 3.0.2'
-  s.add_runtime_dependency 'rugged',  '~> 0.21',  '>= 0.21.4'
+  s.add_runtime_dependency 'bcrypt_pbkdf', '~> 1.0'
+  s.add_runtime_dependency 'ed25519', '~> 1.2'
+  s.add_runtime_dependency 'net-ssh', '~> 5'
+  s.add_runtime_dependency 'net-telnet', '~> 0.2'
+  s.add_runtime_dependency 'rugged',  '~> 0.28.0'
+  s.add_runtime_dependency 'slop',    '~> 4.6'
 
-  if defined?(RUBY_VERSION) && RUBY_VERSION > '2.3'
-      s.add_runtime_dependency 'net-telnet', '~> 0'
-  end
-
-  s.add_development_dependency 'pry',      '~> 0'
-  s.add_development_dependency 'bundler',  '~> 1.10'
-  s.add_development_dependency 'rake',     '~> 10.0'
+  s.add_development_dependency 'bundler', '~> 2.0'
+  s.add_development_dependency 'codecov' if ENV['CI'] == 'true'
+  s.add_development_dependency 'git',      '~> 1'
   s.add_development_dependency 'minitest', '~> 5.8'
   s.add_development_dependency 'mocha',    '~> 1.1'
-  s.add_development_dependency 'git',      '~> 1'
+  s.add_development_dependency 'pry',      '~> 0'
+  s.add_development_dependency 'rake',     '~> 10.0'
+  s.add_development_dependency 'rubocop',  '~> 0.80.0'
+  s.add_development_dependency 'simplecov'
 end