oxidized 0.35.0 → 0.37.0

data/lib/oxidized/model/tplink.rb

@@ -48,19 +48,17 @@ class TPLink < Oxidized::Model
     lines[0..lines.index("end\n")].join
   end
 
+  macro :enable, regex: /^[pP]assword:/
+
   cfg :telnet, :ssh do
     username /^User ?[nN]ame:/
     password /^\r?Password:/
+    newline "\r\n"
   end
 
   cfg :telnet, :ssh do
     post_login do
-      if vars(:enable) == true
-        cmd "enable"
-      elsif vars(:enable)
-        cmd "enable", /^[pP]assword:/
-        cmd vars(:enable)
-      end
+      cmd 'terminal length 0'
     end
 
     pre_logout do

data/lib/oxidized/model/truenas.rb

@@ -5,14 +5,74 @@ class TrueNAS < Oxidized::Model
 
   cmd('uname -a') { |cfg| comment cfg }
   cmd('cat /etc/version') { |cfg| comment cfg }
-  cmd('sqlite3 "file:///data/freenas-v1.db?mode=ro&immutable=1" .dump') do |cfg|
+
+  # for TrueNAS SCALE machines, make sure the user you use to connect can run
+  # this command, or if needed, with passwordless sudo. Try putting this in
+  # /etc/sudoers
+  #    oxidized ALL=(ALL) NOPASSWD: /usr/bin/find /mnt/.ix-apps/app_configs *, /usr/bin/sqlite3 -readonly file\:/data/freenas-v1.db *
+
+  cmd("sqlite3 -readonly 'file:/data/freenas-v1.db' .dump") do |cfg|
+    if cfg.include? "Error: unable to open database"
+      # retry with sudo
+      cfg = cmd("sudo sqlite3 -readonly 'file:/data/freenas-v1.db' .dump")
+    end
     cfg.lines.reject do |line|
-      line.match(/^INSERT INTO storage_replication /) ||
+      line.match(/^INSERT INTO storage_replication /) || # ignore storage_replication because repl_status field changes on every run
         line.match(/^INSERT INTO system_alert /) || # ignore system alerts in db
-        line.match(/^INSERT INTO sqlite_sequence VALUES\('system_alert',/) # ignore system alerts in db
+        line.match(/^INSERT INTO sqlite_sequence VALUES\('system_alert',/) || # ignore system alerts in db
+        line.match(/^INSERT INTO tasks_cloudsync /) # ignore cloudsync tasks because job field changes on every run
     end.join
   end
 
+  post do
+    filter_column("storage_replication", "repl_state")
+  end
+
+  post do
+    filter_column("tasks_cloudsync", "job")
+  end
+
+  post do
+    collect_ixapps_configurations
+  end
+
+  def filter_column(table_name, column_name)
+    sqlite_cmd = "sqlite3 -readonly 'file:/data/freenas-v1.db'"
+
+    # This SQL command will create a SELECT query with all columns except the one we want to skip.
+    generate_select_cmd = "select 'select ' || group_concat(name,', ') || ' FROM #{table_name};' FROM pragma_table_info('#{table_name}') WHERE name != '#{column_name}';"
+
+    select_stmt = cmd("#{sqlite_cmd} \"#{generate_select_cmd}\"")
+    if select_stmt.include? "Error: unable to open database"
+      # retry with sudo
+      sqlite_cmd = "sudo #{sqlite_cmd}"
+      select_stmt = cmd("#{sqlite_cmd} \"#{generate_select_cmd}\"")
+    end
+
+    insert_cmds = "-header '.mode insert #{table_name}' '#{select_stmt}'"
+    cmd("#{sqlite_cmd} #{insert_cmds}") do |cfg|
+      if cfg.include? "INSERT"
+        # Don't add a COMMIT; if the query came up with no rows
+        cfg + "COMMIT;\n"
+      end
+    end
+  end
+
+  def collect_ixapps_configurations
+    config_command = <<~'CMDEND'
+      if [ -d /mnt/.ix-apps ];
+      then
+        sudo find /mnt/.ix-apps/app_configs
+        \( -name "app.yaml" -or -name "user_config.yaml" -or -name "metadata.yaml" \)
+        -printf "\n\n#######################\n# %p\n#######################\n"
+        -exec cat {} \; ;
+      else
+        echo "# No Apps configuration found in /mnt/.ix-apps";
+      fi
+    CMDEND
+    cmd(config_command.gsub("\n", ""))
+  end
+
   cfg :ssh do
     exec true # don't run shell, run each command in exec channel
   end

data/lib/oxidized/model/voss.rb

@@ -23,6 +23,9 @@ class Voss < Oxidized::Model
     cfg.gsub! /(^((.*)Last Vlan Change(.*):(.*))$)/, 'removed Last Vlan Change'
     cfg.gsub! /(^((.*)Temperature(.*):(.*))$)/, 'removed Temperature'
     cfg.gsub! /(^((.*)Total Power Usage(.*):(.*))$)/, 'removed Total Power Usage'
+    cfg.gsub! /(^((.*)(Tray \d+ Fan \d+)\s+(\w+)\s+(\w+)\s+(.*))$)/, '\3\4      \5      [removed RPM values]'
+    cfg.gsub! /(^((.*)Command Execution Time: (.*))$)/, 'Command Execution Time [removed]'
+    cfg.gsub! /(^((.*)(Ambient\s+\d+|CPU|INTERNAL\s+MAC|SODIMM)\s+\d+\s+\d+(.*))$)/, '\3\4  [removed temperature readings]'
     comment "#{cfg}\n"
   end
 

data/lib/oxidized/model/vyos.rb

@@ -15,8 +15,11 @@ class Vyos < Oxidized::Model
   cmd :secret do |cfg|
     cfg.gsub! /secret (\S+).*/, 'secret <secret removed>'
     cfg.gsub! /password (\S+).*/, 'password <secret removed>'
-    cfg.gsub! /community (\S+)/, 'community <secret removed>'
+    cfg.gsub! /snmp community (\S+)/, 'snmp community <secret removed>'
+    cfg.gsub! /snmp trap-target ([^\s\\]*) community (\S+)/, 'snmp trap-target \1 community <secret removed>'
+    cfg.gsub! /preshared-key (\S+).*/, 'preshared-key <secret removed>'
     cfg.gsub! /private key (\S+).*/, 'private key <secret removed>'
+    cfg.gsub! /private-key (\S+).*/, 'private-key <secret removed>'
     # password in URLs like protocol://user:password@domain.tld/
     cfg.gsub! /([a-z]+:\/\/[^:\s]+:)\S+@/, '\1<secret removed>@'
     cfg

data/lib/oxidized/node.rb

@@ -8,8 +8,9 @@ module Oxidized
     include SemanticLogger::Loggable
 
     attr_reader :name, :ip, :model, :input, :output, :group, :auth, :prompt, :timeout, :vars, :last, :repo
-    attr_accessor :running, :user, :email, :msg, :from, :stats, :retry, :err_type, :err_reason
+    attr_accessor :running, :user, :email, :msg, :from, :stats, :retry, :err_type, :err_reason, :nexted
     alias running? running
+    alias nexted? nexted
 
     # opt is a hash with the node parameters given in the source (:name, :group, :ip...)
     def initialize(opt)
@@ -34,6 +35,7 @@ module Oxidized
       @repo = resolve_repo opt
       @err_type = nil
       @err_reason = nil
+      @nexted = false
 
       # model instance needs to access node instance
       @model.node = self
@@ -41,37 +43,37 @@ module Oxidized
 
     def run
       status = :fail
-      config = nil
-      @input.each do |input|
-        # don't try input if model is missing config block, we may need strong config to class_name map
-        cfg_name = input.to_s.split('::').last.downcase
-        next unless @model.cfg[cfg_name] && (not @model.cfg[cfg_name].empty?)
-
-        @model.input = input = input.new
-        if (config = run_input(input))
-          logger.debug "#{input.class.name} ran for #{name} successfully"
-          status = :success
-          break
+      config = Oxidized::Model::Outputs.new
+      input_sequence = @model.class.input_sequence(@input)
+
+      input_sequence.each do |sequence|
+        status = :fail
+        sequence_config = nil
+        sequence.each do |input|
+          @model.input = input = input.new
+          if (sequence_config = run_input(input))
+            logger.debug "#{input.class.name} ran for #{name} successfully"
+            status = :success
+            break
+          else
+            logger.debug "#{input.class.name} failed for #{name}"
+            status = :no_connection
+          end
+        end
+        if status == :success
+          config.merge! sequence_config
         else
-          logger.debug "#{input.class.name} failed for #{name}"
-          status = :no_connection
+          config = nil
+          break
         end
       end
-      logger.error "No suitable input found for #{name}" unless @model.input
 
       @model.input = nil
       [status, config]
     end
 
     def run_input(input)
-      rescue_fail = {}
-      [input.class::RESCUE_FAIL, input.class.superclass::RESCUE_FAIL].each do |hash|
-        hash.each do |level, errors|
-          errors.each do |err|
-            rescue_fail[err] = level
-          end
-        end
-      end
+      rescue_fail = input.class.rescue_fail
       begin
         input.connect(self) && input.get
       rescue *rescue_fail.keys => err

data/lib/oxidized/nodes.rb

@@ -88,6 +88,8 @@ module Oxidized
         n.from = opt['from']
         # set last job to nil so that the node is picked for immediate update
         n.last = nil
+        # set nexted to true so that the node will not be skipped with interval 0
+        n.nexted = true
         put n
         jobs.increment if Oxidized.config.next_adds_job?
       end

data/lib/oxidized/output/file.rb

@@ -21,8 +21,14 @@ module Oxidized
 
       # node: node name (String)
       # outputs: Oxidized::Models::Outputs
-      # opts: hash of node vars
+      # opts: dict of optional parameters:
+      #  - group: node group
+      #  - significant_changes:
+      #      nil / not set / true  -> store as usual
+      #      false                 -> do not store
       def store(node, outputs, opt = {})
+        return false if opt[:significant_changes] == false
+
         file = ::File.expand_path @cfg.directory
         file = ::File.join ::File.dirname(file), opt[:group] if opt[:group]
         FileUtils.mkdir_p file

data/lib/oxidized/output/git.rb

@@ -35,6 +35,16 @@ module Oxidized
         end
       end
 
+      # file: node name (String)
+      # outputs: Oxidized::Models::Outputs
+      # opts: dict of optional parameters:
+      #  - msg: commit message
+      #  - email: committer email
+      #  - user: committer name
+      #  - group: node group
+      #  - significant_changes:
+      #      nil / not set / true: store as usual
+      #      false: skip general config, only store configs where type != nil
       def store(file, outputs, opt = {})
         @msg   = opt[:msg]
         @user  = opt[:user]  || @cfg.user
@@ -58,7 +68,7 @@ module Oxidized
           update type_repo, file, type_cfg
         end
 
-        update repo, file, outputs.to_cfg
+        update repo, file, outputs.to_cfg unless opt[:significant_changes] == false
       end
 
       # Returns the configuration of group/node_name

data/lib/oxidized/output/gitcrypt.rb

@@ -86,7 +86,7 @@ module Oxidized
           update type_repo, file, type_cfg
         end
 
-        update repo, file, outputs.to_cfg
+        update repo, file, outputs.to_cfg unless opt[:significant_changes] == false
       end
 
       def fetch(node, group)

data/lib/oxidized/output/http.rb

@@ -26,9 +26,18 @@ module Oxidized
         @commitref = nil
         uri = URI.parse @cfg.url
         http = Net::HTTP.new uri.host, uri.port
-        # http.use_ssl = true if uri.scheme = 'https'
-        req = Net::HTTP::Post.new(uri.request_uri, 'Content-Type' => 'application/json')
-        req.basic_auth @cfg.user, @cfg.password
+
+        # if uri scheme is https, enable ssl and set verify mode
+        if uri.scheme == "https"
+          http.use_ssl = true
+          http.verify_mode = @cfg.ssl_verify? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+        end
+
+        headers = @cfg.headers? ? @cfg.headers : {}
+
+        req = Net::HTTP::Post.new(uri.request_uri, headers.merge('Content-Type' => 'application/json'))
+        req.basic_auth(@cfg.user, @cfg.password) if @cfg.user? && @cfg.password?
+
         req.body = generate_json(node, outputs, opt)
         response = http.request req
 

data/lib/oxidized/source/csv.rb

@@ -47,6 +47,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: data,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         nodes

data/lib/oxidized/source/jsonfile.rb

@@ -54,6 +54,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         nodes

data/lib/oxidized/source/sql.rb

@@ -46,6 +46,11 @@ module Oxidized
           end
           keys[:vars] = vars unless vars.empty?
 
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node.to_hash,
+                                                      context:  self)
+          next if keys.nil?
+
           nodes << keys
         end
         db.disconnect

data/lib/oxidized/version.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Oxidized
-  VERSION = '0.35.0'
-  VERSION_FULL = '0.35.0'
+  VERSION = '0.37.0'
+  VERSION_FULL = '0.37.0'
   def self.version_set
     version_full = %x(git describe --tags).chop rescue ""
     version      = %x(git describe --tags --abbrev=0).chop rescue ""

data/lib/oxidized/worker.rb

@@ -23,11 +23,11 @@ module Oxidized
                      "#{@jobs_done} of #{@nodes.size}"
         # ask for next node in queue non destructive way
         nextnode = @nodes.first
-        unless nextnode.last.nil?
-          # Set unobtainable value for 'last' if interval checking is disabled
-          last = Oxidized.config.interval.zero? ? Time.now.utc + 10 : nextnode.last.end
-          break if last + Oxidized.config.interval > Time.now.utc
-        end
+        break if Oxidized.config.interval.zero? && !nextnode.nexted?
+
+        nextnode.nexted = false
+        break if !nextnode.last.nil? && (nextnode.last.end + Oxidized.config.interval > Time.now.utc)
+
         # shift nodes and get the next node
         node = @nodes.get
         node.running? ? next : node.running = true
@@ -64,21 +64,43 @@ module Oxidized
 
     private
 
+    def significant_changes?(job, output)
+      node = job.node
+      model = node.model
+      return true unless model.vars(:output_store_mode) == "on_significant"
+
+      unless output.respond_to?(:fetch)
+        logger.error("Detection of significant changes needs an output " \
+                     "capable of fetching the last configuration")
+        return true
+      end
+
+      old = model.significant_changes output.fetch(node, node.group)
+      new = model.significant_changes job.config.to_cfg
+      if old == new
+        logger.debug "No significant change on node #{node.name}"
+        false
+      else
+        true
+      end
+    end
+
     def process_success(node, job)
       @jobs_done += 1 # needed for :nodes_done hook
-      Oxidized.hooks.handle :node_success, node: node,
-                                           job:  job
+      Oxidized.hooks.node_success(node: node, job: job)
       msg = "update #{node.group}/#{node.name}"
       msg += " from #{node.from}" if node.from
       msg += " with message '#{node.msg}'" if node.msg
       output = node.output.new
-      if output.store node.name, job.config,
-                      msg: msg, email: node.email, user: node.user, group: node.group
+
+      significant_changes = significant_changes?(job, output)
+      if output.store(node.name, job.config,
+                      msg: msg, email: node.email, user: node.user,
+                      group: node.group,
+                      significant_changes: significant_changes)
         node.modified
         logger.info "Configuration updated for #{node.group}/#{node.name}"
-        Oxidized.hooks.handle :post_store, node:      node,
-                                           job:       job,
-                                           commitref: output.commitref
+        Oxidized.hooks.post_store(node: node, job: job, commitref: output.commitref)
       end
       node.reset
     end
@@ -97,8 +119,7 @@ module Oxidized
         @jobs_done += 1
         msg += ", retries exhausted, giving up"
         node.retry = 0
-        Oxidized.hooks.handle :node_fail, node: node,
-                                          job:  job
+        Oxidized.hooks.node_fail(node: node, job: job)
       end
       logger.warn msg
     end
@@ -110,7 +131,7 @@ module Oxidized
 
     def run_done_hook
       logger.debug "Running :nodes_done hook"
-      Oxidized.hooks.handle :nodes_done
+      Oxidized.hooks.nodes_done
     rescue StandardError => e
       # swallow the hook erros and continue as normal
       logger.error e.message

data/lib/refinements.rb

@@ -32,6 +32,24 @@ module Refinements
       # rubocop:enable Naming/MemoizedInstanceVariableName
     end
 
+    # keeps lines matching any pattern (String or Regexp)
+    def keep_lines(patterns)
+      each_line.select do |line|
+        patterns.any? do |pattern|
+          pattern.is_a?(Regexp) ? line =~ pattern : line.include?(pattern)
+        end
+      end.join
+    end
+
+    # remove lines matching any pattern (String or Regexp)
+    def reject_lines(patterns)
+      each_line.reject do |line|
+        patterns.any? do |pattern|
+          pattern.is_a?(Regexp) ? line =~ pattern : line.include?(pattern)
+        end
+      end.join
+    end
+
     # Initializes the String instance variables from another String instance
     # when the given str is an instance of String with Oxidized refinements applied
     def init_from_string(str = '')

data/oxidized.gemspec

@@ -18,47 +18,51 @@ Gem::Specification.new do |s|
 
   s.metadata['rubygems_mfa_required'] = 'true'
 
-  s.required_ruby_version = '>= 3.1'
+  # Ruby version strategy
+  #
+  # We try to keep the minimum version of Ruby to match supported LTS versions
+  # of common Linux distributions, as long as the maintenance effort remains
+  # reasonable.
+  s.required_ruby_version = '>= 3.0'
 
   # Gemspec strategy
   #
   # For dependency and optional dependencies, we try to set the minimal
-  # dependency lower than the Ubuntu Noble or Debian Bookworm package version,
+  # dependency lower or equal to the Debian 13 "Trixie" package version,
   # so that native packages can be used.
-  # We limit the maximal version so that dependabot can warn about new versions
-  # and we can test them before activating them in Oxidized.
-  #
-  # development dependencies are set to the latest minor version of a library
-  # and updated after having tested them
+  # We limit the maximal minor version so that dependabot can warn about new
+  # major versions and we can test them before activating them in Oxidized.
 
   s.add_dependency 'asetus',               '~> 0.4'
   s.add_dependency 'bcrypt_pbkdf',         '~> 1.0'
   s.add_dependency 'ed25519',              '~> 1.2'
   s.add_dependency 'net-ftp',              '~> 0.2'
   s.add_dependency 'net-http-digest_auth', '~> 1.4'
-  s.add_dependency 'net-scp',              '~> 4.1'
   s.add_dependency 'net-ssh',              '~> 7.3'
   s.add_dependency 'net-telnet',           '~> 0.2'
   s.add_dependency 'psych',                '~> 5.0'
   s.add_dependency 'rugged',               '~> 1.6'
-  s.add_dependency 'semantic_logger',      '~> 4.17.0'
+  s.add_dependency 'semantic_logger',      '~> 4.17'
   s.add_dependency 'slop',                 '~> 4.6'
-  s.add_dependency 'syslog',               '~> 0.3.0'
-  s.add_dependency 'syslog_protocol',      '~> 0.9.2'
+  s.add_dependency 'syslog',               '~> 0.3'
+  s.add_dependency 'syslog_protocol',      '~> 0.9'
 
-  # ruby-git 4.0 requests ruby >= 3.2, we stick to ruby >= 3.1 (Ubuntu Noble/Debian Bookworm)
-  s.add_development_dependency 'git',                 '>= 2.0', '< 3.2.0'
-  s.add_development_dependency 'minitest',            '~> 5.26.0'
-  s.add_development_dependency 'mocha',               '~> 2.1'
-  s.add_development_dependency 'pry',                 '~> 0.15.0'
-  s.add_development_dependency 'rake',                '~> 13.0'
-  s.add_development_dependency 'rubocop',             '~> 1.81.0'
-  s.add_development_dependency 'rubocop-minitest',    '~> 0.38.0'
-  s.add_development_dependency 'rubocop-rake',        '~> 0.7.0'
-  s.add_development_dependency 'rubocop-sequel',      '~> 0.4.0'
-  s.add_development_dependency 'simplecov',           '~> 0.22.0'
+  s.add_development_dependency 'minitest',         '>= 5.18', '<7'
+  s.add_development_dependency 'mocha',            '>= 2.1', '<4'
+  s.add_development_dependency 'pry',              '~> 0.15'
+  s.add_development_dependency 'rake',             '~> 13.0'
+  # Rubocop introduces new rules in minor versions, so we limit automatic
+  # updates to patches
+  s.add_development_dependency 'rubocop',          '~> 1.86.0'
+  s.add_development_dependency 'rubocop-minitest', '~> 0.39.1'
+  s.add_development_dependency 'rubocop-rake',     '~> 0.7.0'
+  s.add_development_dependency 'rubocop-sequel',   '~> 0.4.0'
+  s.add_development_dependency 'simplecov',        '~> 0.22'
 
   # Dependencies on optional libraries, used for unit tests & development
-  s.add_development_dependency 'oxidized-web',        '>= 0.17.1'
-  s.add_development_dependency 'sequel',              '>= 5.63.0', '< 5.100.0'
+  s.add_development_dependency 'git',              '>= 2.0', '< 5'
+  s.add_development_dependency 'net-scp',          '~> 4.1'
+  s.add_development_dependency 'net-tftp',         '~> 0.1.0'
+  s.add_development_dependency 'oxidized-web',     '>= 0.17.1'
+  s.add_development_dependency 'sequel',           '>= 5.63.0', '< 6'
 end