oxidized 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c4e125d54180398e4fd35652a8ca384c85d451ec
-  data.tar.gz: 46cef914c85dcaadfafcbde310d7c9bbc3707f0b
+  metadata.gz: c8b4c0fb1595706703f57f1d8235cb0521186931
+  data.tar.gz: ec43f93a8c891a3818d36daf7376650da1b4cc59
 SHA512:
-  metadata.gz: 0261cf513efbab1d3bc06ff184c72e70d7371256628441501e2b1d2a44796531091f4fba0eee3b93130993030c4c3e1d78d8c33db01727c17af1760337640f32
-  data.tar.gz: 0662f963ddcbd6da7ab8e6e1cd12ef723c88511cd69ed4673e58e66f00657287cc39c6aec9b0e66bb4c2b441474aa76a67f5bab3a8b5ed7e16ddd1c845236272
+  metadata.gz: 31482aac75b5c83ea25f2e8d3c92ab86f09813caca3e68612a2a719bda2c142bacdf95e29094abf87b0c5f3fa49cfb8fc4a61cd6c4a968cc6f2e60e276282197
+  data.tar.gz: 7a41f80ef435e574734ec761b32c5f86849d4cea2425cdc40694809e59a7087b3618ab174f92d37ef6ab86567d9cda029649c123829b84f5160600db90b9c0da

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+# 0.4.0
+- FEATURE: allow setting IP address in addition to name in source (SQL/CSV)
+- FEATURE: approximate how long it takes to get node from larger view than 1
+- FEATURE: unconditionally start new job if too long has passed since previous start
+- FEATURE: add enable to Arista EOS model
+- FEATURE: add rugged dependency in gemspec
+- FEATURE: log prompt detection failures
+- BUGFIX: xos while using telnet (by @fhibler)
+- BUGFIX: ironware logout on some models (by @fhibler)
+- BUGFIX: allow node to be removed while it is being collected
+- BUGFIX: if model returns non string value, return empty string
+- BUGFIX: better prompt for Arista EOS model (by @rodecker)
+- BUGFIX: improved configuration handling for Arista EOS model (by @rodecker) 
+
 # 0.3.0
 - FEATURE: *FIXME* bunch of stuff I did for richih, docs needed
 - FEATURE: ComWare model (by erJasp)

data/README.md

@@ -71,7 +71,6 @@ Install all required packages and gems.
 
 ```shell
 apt-get install ruby ruby-dev libsqlite3-dev libssl-dev pkg-config cmake
-gem install rugged
 gem install oxidized
 gem install oxidized-script oxidized-web # if you don't install oxidized-web, make sure you remove "rest" from your config
 ```
@@ -193,7 +192,7 @@ source:
       model: 1
       username: 2
       password: 3
-    var_map:
+    vars_map:
       enable: 4
 ```
 
@@ -213,7 +212,7 @@ source:
       model: model
       username: username
       password: password
-    var_map:
+    vars_map:
       enable: enable
 ```
 
@@ -279,7 +278,7 @@ source:
       model: 1
       username: 2
       password: 3
-    var_map:
+    vars_map:
       enable: 4
 model_map:
   cisco: ios

data/Rakefile

@@ -3,7 +3,7 @@ begin
   require 'bundler'
   # Bundler.setup
 rescue LoadError
-  warn 'bunler missing'
+  warn 'bundler missing'
 end
 
 gemspec = eval(File.read(Dir['*.gemspec'].first))

data/extra/nagios_check_failing_nodes.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+## contrib via https://github.com/ytti/oxidized/issues/67
+
+require 'open-uri'
+require 'json'
+
+critical = false
+critical_nodes = []
+
+json = JSON.load(open("http://localhost:8888/nodes.json"))
+json.each do |node|
+  if node['last']['status'] != 'success'
+    critical_nodes << node['name']
+    critical = true
+  end
+end
+
+if critical
+  puts 'Unable to backup: ' + critical_nodes.join(' ')
+  exit 2
+else
+  puts 'Backup of all nodes completed successfully.'
+  exit 0
+end

data/lib/oxidized/core.rb

@@ -17,6 +17,7 @@ module Oxidized
       Oxidized.mgr = Manager.new
       nodes        = Nodes.new
       @worker      = Worker.new nodes
+      trap('HUP') { nodes.load }
       if CFG.rest?
         begin
           require 'oxidized/web'

data/lib/oxidized/input/input.rb

@@ -1,4 +1,5 @@
 module Oxidized
+  class PromptUndetect < OxidizedError; end
   class Input
     include Oxidized::Config::Vars
 
@@ -8,6 +9,7 @@ module Oxidized
       ],
       :warn => [
         IOError,
+        PromptUndetect,
         Timeout::Error,
         Errno::ECONNRESET,
         Errno::EHOSTUNREACH,

data/lib/oxidized/input/ssh.rb

@@ -26,7 +26,11 @@ module Oxidized
                             :paranoid => secure
       unless @exec
         shell_open @ssh
-        @username ? shell_login : expect(@node.prompt)
+        begin
+          @username ? shell_login : expect(@node.prompt)
+        rescue Timeout::Error
+          raise PromptUndetect, [ @output, 'not matching configured prompt', @node.prompt ].join(' ')
+        end
       end
       connected?
     end

data/lib/oxidized/input/telnet.rb

@@ -20,7 +20,11 @@ module Oxidized
       @telnet.puts @node.auth[:username]
       expect password
       @telnet.puts @node.auth[:password]
-      expect @node.prompt
+      begin
+        expect @node.prompt
+      rescue Timeout::Error
+        raise PromptUndetect, [ 'unable to detect prompt:', @node.prompt ].join(' ')
+      end
     end
 
     def connected?

data/lib/oxidized/jobs.rb

@@ -1,24 +1,46 @@
 module Oxidized
   class Jobs < Array
-    attr_accessor :interval, :duration, :max, :want
+    AVERAGE_DURATION  = 5   # initially presume nodes take 5s to complete
+    MAX_INTER_JOB_GAP = 300 # add job if more than X from last job started
+    attr_accessor :interval, :max, :want
+
     def initialize max, interval, nodes
       @max       = max
-      #@interval  = interval * 60
       @interval  = interval
       @nodes     = nodes
-      @duration  = 4
-      new_count
+      @last      = Time.now.utc
+      @durations = Array.new @nodes.size, AVERAGE_DURATION
+      duration AVERAGE_DURATION
       super()
     end
+
+    def push arg
+      @last = Time.now.utc
+      super
+    end
+
     def duration last
-      @duration = (@duration + last) / 2
+      @durations.push(last).shift
+      @duration = @durations.inject(:+).to_f / @nodes.size #rolling average
       new_count
     end
+
     def new_count
       @want = ((@nodes.size * @duration) / @interval).to_i
       @want = 1 if @want < 1
       @want = @nodes.size if @want > @nodes.size
       @want = @max if @want > @max
     end
+
+    def work
+      # if   a) we want less or same amount of threads as we now running
+      # and  b) we want less threads running than the total amount of nodes
+      # and  c) there is more than MAX_INTER_JOB_GAP since last one was started
+      # then we want one more thread (rationale is to fix hanging thread causing HOLB)
+      if @want <= size and @want < @nodes.size
+        @want +=1 if (Time.now.utc - @last) > MAX_INTER_JOB_GAP
+      end
+    end
+
   end
 end

data/lib/oxidized/model/eos.rb

@@ -1,14 +1,13 @@
 class EOS < Oxidized::Model
 
   # Arista EOS model #
-  # need to add telnet support here .. #
 
-  prompt /^[^\(]+\([^\)]+\)#/
+  prompt /^.+[#>]\s?$/
 
   comment  '! '
 
   cmd :all do |cfg|
-     cfg.each_line.to_a[2..-2].join
+     cfg.each_line.to_a[1..-2].join
   end
 
   cmd :secret do |cfg|
@@ -26,7 +25,17 @@ class EOS < Oxidized::Model
   end
 
   cfg :telnet, :ssh do
+    if vars :enable
+      post_login do
+        send "enable\n"
+        expect /[pP]assword:\s?$/
+        send vars(:enable) + "\n"
+        expect /^.+[#>]\s?$/
+      end
+      post_login 'terminal length 0'
+    end
     pre_logout 'exit'
   end
 
 end
+

data/lib/oxidized/model/ironware.rb

@@ -27,7 +27,7 @@ class IronWare < Oxidized::Model
 
   cfg :telnet, :ssh do
     post_login 'skip-page-display'
-    pre_logout 'exit'
+    pre_logout 'logout'
   end
 
 end

data/lib/oxidized/model/model.rb

@@ -133,10 +133,10 @@ module Oxidized
         outputs << out
       end
       procs[:pre].each do |pre_proc|
-        outputs.unshift Oxidized::String.new(instance_eval(&pre_proc))
+        outputs.unshift process_cmd_output(instance_eval(&pre_proc), '')
       end
       procs[:post].each do |post_proc|
-        outputs << Oxidized::String.new(instance_eval(&post_proc))
+        outputs << process_cmd_output(instance_eval(&post_proc), '')
       end
       outputs
     end
@@ -152,9 +152,8 @@ module Oxidized
     private
 
     def process_cmd_output output, name
-      if output.class != Oxidized::String
-        output = Oxidized::String.new output
-      end
+      output = Oxidized::String.new output if ::String === output
+      output = Oxidized::String.new '' unless Oxidized::String === output
       output.set_cmd(name)
       output
     end

data/lib/oxidized/model/xos.rb

@@ -29,7 +29,7 @@ class XOS < Oxidized::Model
 
   cfg :telnet do
     username /^login:/
-    password /^passowrd:/
+    password /^\r*password:/
   end
 
   cfg :telnet, :ssh do

data/lib/oxidized/node.rb

@@ -10,7 +10,8 @@ module Oxidized
     alias :running? :running
     def initialize opt
       @name           = opt[:name]
-      @ip             = Resolv.getaddress @name
+      @ip             = IPAddr.new(opt[:ip]).to_s rescue nil
+      @ip           ||= Resolv.new.getaddress @name
       @group          = opt[:group]
       @input          = resolve_input opt
       @output         = resolve_output opt

data/lib/oxidized/nodes.rb

@@ -1,6 +1,6 @@
 module Oxidized
- require 'oxidized/node'
  require 'ipaddr'
+ require 'oxidized/node'
  class Oxidized::NotSupported < OxidizedError; end
  class Oxidized::NodeNotFound < OxidizedError; end
   class Nodes < Array
@@ -23,8 +23,8 @@ module Oxidized
             Log.error "node %s is not resolvable, raised %s with message '%s'" % [node, err.class, err.message]
           end
         end
-        Log.info "Loaded #{size} nodes"
         size == 0 ? replace(new) : update_nodes(new)
+	Log.info "Loaded #{size} nodes"
       end
     end
 

data/lib/oxidized/output/git.rb

@@ -1,5 +1,6 @@
 module Oxidized
 class Git < Output
+  class GitError < OxidizedError; end
   begin
     gem 'rugged', '~> 0.21.0'
     require 'rugged'
@@ -71,8 +72,12 @@ class Git < Output
     end
     repo = Rugged::Repository.new repo
     update_repo repo, file, data, @msg, @user, @email
-  rescue Rugged::OSError, Rugged::RepositoryError
-    Rugged::Repository.init_at repo, :bare
+  rescue Rugged::OSError, Rugged::RepositoryError => open_error
+    begin
+      Rugged::Repository.init_at repo, :bare
+    rescue => create_error
+      raise GitError, "first '#{open_error.message}' was raised while opening git repo, then '#{create_error.message}' was while trying to create git repo"
+    end
     retry
   end
 

data/lib/oxidized/worker.rb

@@ -7,10 +7,12 @@ module Oxidized
       @jobs    = Jobs.new CFG.threads, CFG.interval, @nodes
       Thread.abort_on_exception = true
     end
+
     def work
       ended = []
       @jobs.delete_if { |job| ended << job if not job.alive? }
       ended.each      { |job| process job }
+      @jobs.work
       while @jobs.size < @jobs.want
         Log.debug "Jobs #{@jobs.size}, Want: #{@jobs.want}"
         # ask for next node in queue non destructive way
@@ -24,6 +26,7 @@ module Oxidized
         @jobs.push Job.new node
       end
     end
+
     def process job
       node = job.node
       node.last = job
@@ -49,6 +52,9 @@ module Oxidized
         end
         Log.warn msg
       end
+    rescue NodeNotFound
+      Log.warn "#{node.name} not found, removed while collecting?"
     end
+
   end
 end

data/oxidized.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name              = 'oxidized'
-  s.version           = '0.3.0'
+  s.version           = '0.4.0'
   s.licenses          = %w( Apache-2.0 )
   s.platform          = Gem::Platform::RUBY
   s.authors           = [ 'Saku Ytti', 'Samer Abdel-Hafez' ]
@@ -17,4 +17,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'asetus',  '~> 0.1'
   s.add_runtime_dependency 'slop',    '~> 3.5'
   s.add_runtime_dependency 'net-ssh', '~> 2.8'
+  s.add_runtime_dependency 'rugged',  '~> 0.21.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: oxidized
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Saku Ytti
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-13 00:00:00.000000000 Z
+date: 2015-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: asetus
@@ -53,6 +53,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.8'
+- !ruby/object:Gem::Dependency
+  name: rugged
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.21.4
 description: software to fetch configuration from network devices and store them
 email:
 - saku@ytti.fi
@@ -68,6 +82,7 @@ files:
 - Rakefile
 - TODO.md
 - bin/oxidized
+- extra/nagios_check_failing_nodes.rb
 - extra/oxidized.init
 - extra/rest_client.rb
 - extra/syslog.rb