oxidized 0.35.0 → 0.36.0

data/lib/oxidized/model/model.rb

@@ -1,5 +1,8 @@
 require 'strscan'
 require_relative 'outputs'
+require_relative 'dslsetup'
+require_relative 'dslcommands'
+require_relative 'macros'
 
 module Oxidized
   class Model
@@ -7,6 +10,11 @@ module Oxidized
 
     using Refinements
 
+    # Domain Specific Language for models
+    extend Oxidized::Model::DSLSetup
+    extend Oxidized::Model::DSLCommands
+    extend Oxidized::Model::Macros
+
     include Oxidized::Config::Vars
 
     # rubocop:disable Style/FormatStringToken
@@ -25,6 +33,8 @@ module Oxidized
           klass.instance_variable_set '@comment', nil
           klass.instance_variable_set '@prompt',  nil
           klass.instance_variable_set '@metadata', {}
+          klass.instance_variable_set '@inputs', nil
+
         else # we're subclassing some existing model, take its variables
           instance_variables.each do |var|
             iv = instance_variable_get(var)
@@ -33,136 +43,20 @@ module Oxidized
           end
         end
       end
-
-      def comment(str = "# ")
-        @comment = if block_given?
-                     yield
-                   elsif not @comment
-                     str
-                   else
-                     @comment
-                   end
-      end
-
-      def prompt(regex = nil)
-        @prompt = regex || @prompt
-      end
-
-      def cfg(*methods, **args, &block)
-        [methods].flatten.each do |method|
-          process_args_block(@cfg[method.to_s], args, block)
-        end
-      end
-
-      def cfgs
-        @cfg
-      end
-
-      def cmd(cmd_arg = nil, **args, &block)
-        if cmd_arg.instance_of?(Symbol)
-          process_args_block(@cmd[cmd_arg], args, block)
-        else
-          # Normal command
-          process_args_block(@cmd[:cmd], args,
-                             { cmd: cmd_arg, args: args, block: block })
-        end
-        logger.debug "Added #{cmd_arg} to the commands list"
-      end
-
-      def metadata(position, value = nil, &block)
-        return unless %i[top bottom].include? position
-
-        if block_given?
-          @metadata[position] = block
-        else
-          @metadata[position] = value
-        end
-      end
-
-      def cmds
-        @cmd
-      end
-
-      def expect(regex, **args, &block)
-        process_args_block(@expect, args, [regex, block])
-      end
-
-      def expects
-        @expect
-      end
-
-      def clean(what)
-        case what
-        when :escape_codes
-          ansi_escape_regex = /
-            \r?        # Optional carriage return at start
-            \e         # ESC character - starts escape sequence
-            (?:        # Non-capturing group for different sequence types:
-              # Type 1: CSI (Control Sequence Introducer)
-              \[       # Literal '[' - starts CSI sequence
-              [0-?]*   # Parameter bytes: digits (0-9), semicolon, colon, etc.
-              [ -\/]*  # Intermediate bytes: space through slash characters
-              [@-~]    # Final byte: determines the actual command
-            |          # OR
-              # Type 2: Simple escape
-              [=>]     # Single character commands after ESC
-            )
-            \r?        # Optional carriage return at end
-          /x
-          expect ansi_escape_regex do |data, re|
-            data.gsub re, ''
-          end
-        end
-      end
-
-      # calls the block at the end of the model, prepending the output of the
-      # block to the output string
-      #
-      # @yield expects block which should return [String]
-      # @return [void]
-      def pre(**args, &block)
-        process_args_block(@procs[:pre], args, block)
-      end
-
-      # calls the block at the end of the model, adding the output of the block
-      # to the output string
-      #
-      # @yield expects block which should return [String]
-      # @return [void]
-      def post(**args, &block)
-        process_args_block(@procs[:post], args, block)
-      end
-
-      # @author Saku Ytti <saku@ytti.fi>
-      # @since 0.0.39
-      # @return [Hash] hash proc procs :pre+:post to be prepended/postfixed to output
-      attr_reader :procs
-
-      private
-
-      def process_args_block(target, args, block)
-        if args[:clear]
-          if block.instance_of?(Array)
-            target.reject! { |k, _| k == block[0] }
-            target.push(block)
-          elsif block.instance_of?(Hash)
-            target.reject! { |item| item[:cmd] == block[:cmd] }
-            target.push(block)
-          else
-            target.replace([block])
-          end
-        else
-          method = args[:prepend] ? :unshift : :push
-          target.send(method, block)
-        end
-      end
     end
 
     attr_accessor :input, :node
 
-    def cmd(string, &block)
+    # input specifies to run this command only with this input type
+    # if input is not specified, always run the command
+    def cmd(string, input: nil, &block)
       logger.debug "Executing #{string}"
-      out = @input.cmd(string)
+      out = if input.nil? || input.include?(@input.to_sym)
+              out = @input.cmd(string)
+            else
+              # Do not run this command
+              return ''
+            end
       return false unless out
 
       out = out.b unless Oxidized.config.input.utf8_encoded?
@@ -245,6 +139,7 @@ module Oxidized
       data
     end
 
+    # Get the commands from the model
     def get
       logger.debug 'Collecting commands\' outputs'
       outputs = Outputs.new
@@ -255,7 +150,7 @@ module Oxidized
 
         next if args.include?(:if) && !instance_exec(&args[:if])
 
-        out = cmd command, &block
+        out = cmd command, input: args[:input], &block
         return false unless out
 
         outputs << out
@@ -306,6 +201,13 @@ module Oxidized
       @input.class.to_s.match(/Telnet/) || vars(:ssh_no_exec)
     end
 
+    def significant_changes(config)
+      self.class.cmds[:significant_changes].each do |block|
+        config = instance_exec config, &block
+      end
+      config
+    end
+
     private
 
     def process_cmd_output(output, name)

data/lib/oxidized/model/ndms.rb

@@ -22,4 +22,10 @@ class NDMS < Oxidized::Model
     password /^Password:/
     pre_logout 'exit'
   end
+
+  cmd :significant_changes do |cfg|
+    cfg.reject_lines [
+      'Last change:'
+    ]
+  end
 end

data/lib/oxidized/model/netgear.rb

@@ -6,7 +6,7 @@ class Netgear < Oxidized::Model
   clean :escape_codes
 
   # Handle pager for "show version" on old Netgear models: #2394
-  expect /^--More-- or \(q\)uit$/ do |data, re|
+  expect /^--More--(?: or \(q\)uit)?$/ do |data, re|
     send ' '
     data.sub re, ''
   end
@@ -20,7 +20,7 @@ class Netgear < Oxidized::Model
   end
 
   cfg :telnet do
-    username /^(User:|Applying Interface configuration, please wait ...)/
+    username /^(Username:|User:|Applying Interface configuration, please wait ...)/
     password /^Password:/i
   end
 
@@ -58,9 +58,11 @@ class Netgear < Oxidized::Model
     comment cfg
   end
   cmd 'show running-config' do |cfg|
-    cfg.gsub! /(System Up Time\s+).*/, '\\1 <removed>'
+    cfg.gsub! /(System Up Time:?\s+).*/, '\\1 <removed>'
     cfg.gsub! /(Current SNTP Synchronized Time:).*/, '\\1 <removed>'
     cfg.gsub! /(Current System Time:).*/, '\\1 <removed>'
+    # Remove standalone backspace lines
+    cfg.gsub!(/(?:\r?\n)?\x08(?:\r?\n)?/, "\n")
     cfg
   end
 end

data/lib/oxidized/model/nxos.rb

@@ -25,8 +25,8 @@ class NXOS < Oxidized::Model
   end
 
   cmd 'show inventory all' do |cfg|
-    if cfg.include? "% Invalid parameter detected at '^' marker."
-      # 'show inventory all' isn't supported on older versions (See Issue #3657)
+    if cfg.match? /^% Invalid .* at '\^' marker\./
+      # 'show inventory all' isn't supported on older versions (See Issues #3657, #3779)
       cfg = cmd 'show inventory'
     end
     comment cfg

data/lib/oxidized/model/outputs.rb

@@ -31,6 +31,11 @@ module Oxidized
         @outputs.map { |out| out.type }.uniq.compact
       end
 
+      def merge!(other)
+        other.all.each { |o| @outputs << o }
+        self
+      end
+
       private
 
       def initialize

data/lib/oxidized/model/perle.rb

@@ -5,7 +5,8 @@ class Perle < Oxidized::Model
   comment '! '
 
   cmd :all do |cfg|
-    cfg.cut_both
+    cfg = cfg.cut_both
+    cfg.delete "\r"
   end
 
   cmd 'show version verbose' do |cfg|
@@ -17,17 +18,22 @@ class Perle < Oxidized::Model
   end
 
   cmd 'show interfaces transceiver' do |cfg|
-    out = []
-    cfg.each_line do |line|
-      out << line if line =~ /SFP Information/
-      out << line if line =~ /Vendor Name/
-      out << line if line =~ /Vendor Serial Number/
-    end
-    comment out.join + "\n"
+    cfg = cfg.keep_lines [
+      'SFP Information',
+      'Vendor Name',
+      'Vendor Serial Number'
+    ]
+    comment cfg + "\n"
   end
 
   cmd 'show running-config'
 
+  cmd :significant_changes do |cfg|
+    cfg.reject_lines [
+      /^tacacs-server key 7 \$0\$\S+==$/
+    ]
+  end
+
   cfg :ssh do
     post_login 'terminal length 0'
     pre_logout 'exit'

data/lib/oxidized/model/smartbyte.rb

@@ -0,0 +1,48 @@
+class SmartByte < Oxidized::Model
+  using Refinements
+
+  comment '! '
+
+  cmd :secret do |cfg|
+    cfg.gsub!(/group (\S+) v2c/, 'group <hidden> v2c')
+    cfg.gsub!(/community (\S+)/, 'community <hidden>')
+    cfg
+  end
+
+  cmd :all do |cfg|
+    cfg = cfg.delete("\r")
+    cfg.cut_tail
+  end
+
+  cmd 'show running-config'
+
+  cmd 'show version' do |cfg|
+    comment cfg
+  end
+
+  cmd 'show interface optical-transceiver info' do |cfg|
+    cfg.gsub! /^\|Transceiver current alarm information:           \|\s+\+-------------------------------------------------\+.*?\s+\+-------------------------------------------------\+\s+/m, ''
+    comment cfg
+  end
+
+  cmd 'show power' do |cfg|
+    comment cfg
+  end
+
+  cfg :telnet do
+    username /^.*? login:/
+    password /^Password:/
+  end
+
+  cfg :telnet, :ssh do
+    post_login do
+      if vars(:enable)
+        cmd "enable", /^[pP]assword:/
+        cmd vars(:enable)
+      end
+    end
+
+    post_login 'terminal length 0'
+    pre_logout 'exit'
+  end
+end

data/lib/oxidized/model/truenas.rb

@@ -5,7 +5,16 @@ class TrueNAS < Oxidized::Model
 
   cmd('uname -a') { |cfg| comment cfg }
   cmd('cat /etc/version') { |cfg| comment cfg }
-  cmd('sqlite3 "file:///data/freenas-v1.db?mode=ro&immutable=1" .dump') do |cfg|
+
+  # for TrueNAS SCALE machines, make sure the user you use to connect can run
+  # this command, or if needed, with passwordless sudo. Try putting this in
+  # /etc/sudoers
+  #    oxidized ALL=(ALL) NOPASSWD: /usr/bin/sqlite3 file\:///data/freenas-v1.db?mode\=ro&immutable\=1 .dump
+
+  cmd("sqlite3 'file:///data/freenas-v1.db?mode=ro&immutable=1' .dump") do |cfg|
+    if cfg.include? "Error: unable to open database"
+      cfg = cmd("sudo sqlite3 'file:///data/freenas-v1.db?mode=ro&immutable=1' .dump")
+    end
     cfg.lines.reject do |line|
       line.match(/^INSERT INTO storage_replication /) ||
         line.match(/^INSERT INTO system_alert /) || # ignore system alerts in db

data/lib/oxidized/model/voss.rb

@@ -23,6 +23,9 @@ class Voss < Oxidized::Model
     cfg.gsub! /(^((.*)Last Vlan Change(.*):(.*))$)/, 'removed Last Vlan Change'
     cfg.gsub! /(^((.*)Temperature(.*):(.*))$)/, 'removed Temperature'
     cfg.gsub! /(^((.*)Total Power Usage(.*):(.*))$)/, 'removed Total Power Usage'
+    cfg.gsub! /(^((.*)(Tray \d+ Fan \d+)\s+(\w+)\s+(\w+)\s+(.*))$)/, '\3\4      \5      [removed RPM values]'
+    cfg.gsub! /(^((.*)Command Execution Time: (.*))$)/, 'Command Execution Time [removed]'
+    cfg.gsub! /(^((.*)(Ambient\s+\d+|CPU|INTERNAL\s+MAC|SODIMM)\s+\d+\s+\d+(.*))$)/, '\3\4  [removed temperature readings]'
     comment "#{cfg}\n"
   end
 

data/lib/oxidized/model/vyos.rb

@@ -15,8 +15,10 @@ class Vyos < Oxidized::Model
   cmd :secret do |cfg|
     cfg.gsub! /secret (\S+).*/, 'secret <secret removed>'
     cfg.gsub! /password (\S+).*/, 'password <secret removed>'
-    cfg.gsub! /community (\S+)/, 'community <secret removed>'
+    cfg.gsub! /snmp community (\S+)/, 'snmp community <secret removed>'
+    cfg.gsub! /preshared-key (\S+).*/, 'preshared-key <secret removed>'
     cfg.gsub! /private key (\S+).*/, 'private key <secret removed>'
+    cfg.gsub! /private-key (\S+).*/, 'private-key <secret removed>'
     # password in URLs like protocol://user:password@domain.tld/
     cfg.gsub! /([a-z]+:\/\/[^:\s]+:)\S+@/, '\1<secret removed>@'
     cfg

data/lib/oxidized/node.rb

@@ -8,8 +8,9 @@ module Oxidized
     include SemanticLogger::Loggable
 
     attr_reader :name, :ip, :model, :input, :output, :group, :auth, :prompt, :timeout, :vars, :last, :repo
-    attr_accessor :running, :user, :email, :msg, :from, :stats, :retry, :err_type, :err_reason
+    attr_accessor :running, :user, :email, :msg, :from, :stats, :retry, :err_type, :err_reason, :nexted
     alias running? running
+    alias nexted? nexted
 
     # opt is a hash with the node parameters given in the source (:name, :group, :ip...)
     def initialize(opt)
@@ -34,6 +35,7 @@ module Oxidized
       @repo = resolve_repo opt
       @err_type = nil
       @err_reason = nil
+      @nexted = false
 
       # model instance needs to access node instance
       @model.node = self
@@ -41,37 +43,37 @@ module Oxidized
 
     def run
       status = :fail
-      config = nil
-      @input.each do |input|
-        # don't try input if model is missing config block, we may need strong config to class_name map
-        cfg_name = input.to_s.split('::').last.downcase
-        next unless @model.cfg[cfg_name] && (not @model.cfg[cfg_name].empty?)
-
-        @model.input = input = input.new
-        if (config = run_input(input))
-          logger.debug "#{input.class.name} ran for #{name} successfully"
-          status = :success
-          break
+      config = Oxidized::Model::Outputs.new
+      input_sequence = @model.class.input_sequence(@input)
+
+      input_sequence.each do |sequence|
+        status = :fail
+        sequence_config = nil
+        sequence.each do |input|
+          @model.input = input = input.new
+          if (sequence_config = run_input(input))
+            logger.debug "#{input.class.name} ran for #{name} successfully"
+            status = :success
+            break
+          else
+            logger.debug "#{input.class.name} failed for #{name}"
+            status = :no_connection
+          end
+        end
+        if status == :success
+          config.merge! sequence_config
         else
-          logger.debug "#{input.class.name} failed for #{name}"
-          status = :no_connection
+          config = nil
+          break
         end
       end
-      logger.error "No suitable input found for #{name}" unless @model.input
 
       @model.input = nil
       [status, config]
     end
 
     def run_input(input)
-      rescue_fail = {}
-      [input.class::RESCUE_FAIL, input.class.superclass::RESCUE_FAIL].each do |hash|
-        hash.each do |level, errors|
-          errors.each do |err|
-            rescue_fail[err] = level
-          end
-        end
-      end
+      rescue_fail = input.class.rescue_fail
       begin
         input.connect(self) && input.get
       rescue *rescue_fail.keys => err

data/lib/oxidized/nodes.rb

@@ -88,6 +88,8 @@ module Oxidized
         n.from = opt['from']
         # set last job to nil so that the node is picked for immediate update
         n.last = nil
+        # set nexted to true so that the node will not be skipped with interval 0
+        n.nexted = true
         put n
         jobs.increment if Oxidized.config.next_adds_job?
       end

data/lib/oxidized/output/file.rb

@@ -21,8 +21,14 @@ module Oxidized
 
       # node: node name (String)
       # outputs: Oxidized::Models::Outputs
-      # opts: hash of node vars
+      # opts: dict of optional parameters:
+      #  - group: node group
+      #  - significant_changes:
+      #      nil / not set / true  -> store as usual
+      #      false                 -> do not store
       def store(node, outputs, opt = {})
+        return false if opt[:significant_changes] == false
+
         file = ::File.expand_path @cfg.directory
         file = ::File.join ::File.dirname(file), opt[:group] if opt[:group]
         FileUtils.mkdir_p file

data/lib/oxidized/output/git.rb

@@ -35,6 +35,16 @@ module Oxidized
         end
       end
 
+      # file: node name (String)
+      # outputs: Oxidized::Models::Outputs
+      # opts: dict of optional parameters:
+      #  - msg: commit message
+      #  - email: committer email
+      #  - user: committer name
+      #  - group: node group
+      #  - significant_changes:
+      #      nil / not set / true: store as usual
+      #      false: skip general config, only store configs where type != nil
       def store(file, outputs, opt = {})
         @msg   = opt[:msg]
         @user  = opt[:user]  || @cfg.user
@@ -58,7 +68,7 @@ module Oxidized
           update type_repo, file, type_cfg
         end
 
-        update repo, file, outputs.to_cfg
+        update repo, file, outputs.to_cfg unless opt[:significant_changes] == false
       end
 
       # Returns the configuration of group/node_name

data/lib/oxidized/output/gitcrypt.rb

@@ -86,7 +86,7 @@ module Oxidized
           update type_repo, file, type_cfg
         end
 
-        update repo, file, outputs.to_cfg
+        update repo, file, outputs.to_cfg unless opt[:significant_changes] == false
       end
 
       def fetch(node, group)

data/lib/oxidized/output/http.rb

@@ -26,9 +26,18 @@ module Oxidized
         @commitref = nil
         uri = URI.parse @cfg.url
         http = Net::HTTP.new uri.host, uri.port
-        # http.use_ssl = true if uri.scheme = 'https'
-        req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
-        req.basic_auth @cfg.user, @cfg.password
+
+        # if uri scheme is https, enable ssl and set verify mode
+        if uri.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = @cfg.ssl_verify? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        end
+
+        headers = @cfg.headers? ? @cfg.headers : {}
+
+        req = Net::HTTP::Post.new(uri.request_uri, headers.merge('Content-Type' => 'application/json'))
+        req.basic_auth(@cfg.user, @cfg.password) if @cfg.user? && @cfg.password?
+
         req.body = generate_json(node, outputs, opt)
         response = http.request req
 

data/lib/oxidized/source/csv.rb

@@ -47,6 +47,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: data,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         nodes

data/lib/oxidized/source/jsonfile.rb

@@ -54,6 +54,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         nodes

data/lib/oxidized/source/sql.rb

@@ -46,6 +46,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node.to_hash,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         db.disconnect

data/lib/oxidized/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Oxidized
-  VERSION = '0.35.0'
-  VERSION_FULL = '0.35.0'
+  VERSION = '0.36.0'
+  VERSION_FULL = '0.36.0'
   def self.version_set
     version_full = %x(git describe --tags).chop rescue ""
     version      = %x(git describe --tags --abbrev=0).chop rescue ""