oxidized 0.35.0 → 0.36.0

data/extra/hooks/modelrulesadvanced.rb

@@ -0,0 +1,168 @@
+# Advanced dynamic model selection hook for Oxidized
+# --------------------------------------------------
+# This hook allows you to assign Oxidized models based on any device attributes
+# (name, vendor, group, type, IP, etc.) using flexible rules defined in the config.
+# It uses the `source_node_transform` event and works with any source (CSV, HTTP, SQL).
+#
+# The hook is designed to be generic: you can specify any field that exists in the
+# node data after source mapping (see `map` section in the source configuration).
+# Rules are evaluated in order; the first matching rule assigns its `model`.
+#
+# ⚠️ IMPORTANT: For the hook to load correctly:
+#   - The filename must match the hook type (e.g., `modelrulesadvanced.rb`).
+#   - The class name must be the CamelCase version of the filename without underscores
+#     (e.g., filename `modelrulesadvanced.rb` → class `Modelrulesadvanced`).
+#   - In the Oxidized config, specify `type: modelrulesadvanced` (the filename without .rb).
+#
+# ⚠️ DOCKER NOTE: This hook was developed and tested in a Docker container.
+#   Ensure that your hooks directory inside the container is
+#   `/home/oxidized/.config/oxidized/hook` (singular "hook"), not "hooks".
+#   In docker-compose, mount your local hooks folder to this path, e.g.:
+#   volumes:
+#     - ./extra/hooks:/home/oxidized/.config/oxidized/hook
+#
+# Example scenario with NetBox:
+#   Suppose your NetBox instance returns devices with the following data (simplified):
+#     gw-001  | Mikrotik | RB750 | ro_ud  | 192.168.0.167/24
+#     gw-002  | Mikrotik | RB750 | ro_ud  | 192.168.0.177/24
+#     gw-003  | Mikrotik | RB750 | ro_bd  | 192.168.0.178/24
+#     gw-004  | Mikrotik | RB750 | switch | 192.168.0.181/24
+#     gw-005  | Cisco    | 2960  | switch | 192.168.0.179/24
+#     gw-006  | Mikrotik | RB750 | switch | 192.168.0.180/24
+#     gw-007  | Arista   | 3456  | ro_ud  | 192.168.0.190/24
+#
+#   Desired model assignment:
+#     - gw-001 (exception by name) → asa
+#     - MikroTik in group `switch` with IP 192.168.0.180/24 → eltex
+#     - All other MikroTik → routeros
+#     - Cisco switches → ios
+#     - Arista devices → eos
+#
+# Example configuration (top-level in Oxidized config):
+#   ---
+#   username: oxidized_ssh_user
+#   password: oxidized_ssh_password
+#
+#   hooks:
+#     modelrulesadvanced:
+#       type: modelrulesadvanced
+#       events: [source_node_transform]
+#       rules:
+#         - description: "Exception: gw-001 uses ASA model"
+#           name: gw-001
+#           model: asa
+#         - description: "Mikrotik switch with IP 192.168.0.180/24 uses eltex"
+#           vendor: Mikrotik
+#           group: switch
+#           ip: 192.168.0.180/24
+#           model: eltex
+#         - description: "All other Mikrotik devices"
+#           vendor: Mikrotik
+#           model: routeros
+#         - description: "Cisco switches"
+#           vendor: Cisco
+#           group: switch
+#           model: ios
+#         - description: "Arista devices"
+#           vendor: Arista
+#           model: eos
+#
+#   resolve_dns: false
+#   interval: 3600
+#   rest: 0.0.0.0:8888
+#   log: /home/oxidized/.config/oxidized/logs/oxidized.log
+#   debug: true   # optional – enables detailed logging from the hook
+#
+#   source:
+#     default: http
+#     http:
+#       url: http://netbox.test/api/dcim/devices/?status=active&has_primary_ip=true
+#       headers:
+#         Authorization: Token YOUR_API_TOKEN
+#       map:
+#         name: name
+#         vendor: device_type.manufacturer.name
+#         type: device_type.model
+#         group: role.slug
+#         ip: primary_ip.address
+#       secure: false
+#       hosts_location: results
+#       pagination: true
+#       pagination_key_name: next
+#
+#   output:
+#     default: file
+#     file:
+#       directory: "/home/oxidized/.config/oxidized/configs"
+#
+#   groups:
+#     ro_ud: {}
+#     ro_bd: {}
+#     switch: {}
+#
+# How it works:
+#   1. Oxidized loads node data from the source (NetBox) and applies the `map`.
+#   2. For each node, the `source_node_transform` event is triggered.
+#   3. This hook receives a context object `ctx` containing:
+#        - `ctx.node` – the mapped node attributes (hash)
+#        - `ctx.node_raw` – the original source record (useful for unmapped fields)
+#   4. The hook iterates through the rules defined in `cfg.rules`.
+#   5. For each rule, it checks that every specified key (except `model` and `description`)
+#      matches the corresponding value in `ctx.node`. Comparison is case‑insensitive and
+#      strips surrounding spaces.
+#   6. The first matching rule assigns its `model` to `ctx.node[:model]`.
+#   7. If no rule matches, the node's `model` remains unchanged.
+#   8. The modified (or original) node is returned; returning `nil` would exclude the node.
+#
+# Notes:
+#   - Rule order is crucial: place more specific rules (e.g., with IP or name) before generic ones.
+#   - The `description` field is optional and only appears in debug logs.
+#   - Any field present in `ctx.node` after mapping can be used as a match key.
+#   - For HTTP sources (NetBox), you can access additional fields via `ctx.node_raw["field"]`.
+#   - Debug logging requires `debug: true` in the global config.
+
+class Modelrulesadvanced < Oxidized::Hook
+  # Validate that the hook configuration contains a 'rules' array.
+  def validate_cfg!
+    raise KeyError, "hook.rules is required" unless cfg.has_key?("rules")
+  end
+
+  # Main hook method – called for each node during source_node_transform event.
+  def run_hook(ctx)
+    node = ctx.node
+    rules = cfg.rules || []
+
+    matched_model = nil
+    rules.each_with_index do |rule, idx|
+      match = true
+      rule.each do |key, value|
+        next if %w[model description].include?(key)
+
+        # Retrieve the corresponding value from the node (symbol or string key)
+        node_value = node[key.to_sym] || node[key.to_s]
+        if node_value.to_s.strip.downcase != value.to_s.strip.downcase
+          match = false
+          break
+        end
+      end
+      next unless match
+
+      matched_model = rule["model"]
+      desc = rule["description"] ? " (#{rule['description']})" : ""
+      logger.debug "ModelRulesAdvanced: rule #{idx + 1}#{desc} matched -> #{matched_model}"
+      break
+    end
+
+    if matched_model
+      old_model = node[:model] || node["model"]
+      node[:model] = matched_model
+      logger.debug "ModelRulesAdvanced: changed model from #{old_model.inspect} to #{matched_model.inspect}"
+    else
+      # rubocop:disable Layout/LineLength
+      logger.debug "ModelRulesAdvanced: no rule matched, keeping existing model: #{node[:model] || node['model'].inspect}"
+      # rubocop:enable Layout/LineLength
+    end
+
+    node
+  end
+end

data/extra/hooks/srcipmap.rb

@@ -0,0 +1,54 @@
+### script in ~/config/oxidized/hook/srcipmap.rb ## or OXDIZED_HOME equivalent
+###
+### router.db:
+### router1:1.1.1.1:cisco:c7200:10.10.10.1:somerole
+### router2:2.2.2.2:juniper:mx80:10.10.10.2:wlc
+### router3:3.3.3.3:juniper:mx2020:10.10.10.3:anotherrole
+###
+### config:
+### source:
+###   default: csv
+###   csv:
+###    file: "/Users/ytti/.config/oxidized/router.db"
+###    delimiter: !ruby/regexp /:/
+###    map:
+###      name: 0
+###      ip: 1
+###      model: 2
+### model_map:
+###   juniper: junos
+###   cisco: ios
+### hooks:
+###   somename:
+###    type: srcipmap
+###    events: ["source_node_transform"]
+###
+###
+###
+### Nodes BEFORE script:
+### {name: "router1", ip: "1.1.1.1", model: "ios"}
+### {name: "router2", ip: "2.2.2.2", model: "junos"}
+### {name: "router3", ip: "3.3.3.3", model: "junos"
+###
+### Nodes AFTER script:
+### {name: "router1", ip: "1.1.1.1", model: "ios"}
+### {name: "router2", ip: "10.10.10.2", model: "junos"}
+### {name: "router3", ip: "3.3.3.3", model: "junos"}
+
+class SrcIpMap < Oxidized::Hook
+  def run_hook(ctx)
+    # node is the node[key] that we'd return without manipulation
+    node = ctx.node
+
+    ## node_raw is source specific, in CSV it is just the field number
+    _platform = ctx.node_raw[3]
+    oob_ip = ctx.node_raw[4]
+    role = ctx.node_raw[5]
+
+    ### the magic
+    node[:ip] = oob_ip if role == 'wlc'
+
+    ### remember to return the manipulated object, nil if you want to ignore loading node
+    node
+  end
+end

data/lib/oxidized/hook/githubrepo.rb

@@ -86,7 +86,7 @@ class GithubRepo < Oxidized::Hook
   private
 
   def credentials(node)
-    Proc.new do |_url, username_from_url, _allowed_types| # rubocop:disable Style/Proc
+    proc do |_url, username_from_url, _allowed_types|
       git_user = cfg.has_key?('username') ? cfg.username : (username_from_url || 'git')
       if cfg.has_key?('password')
         logger.debug "Authenticating using username and password as '#{git_user}'"
@@ -96,6 +96,7 @@ class GithubRepo < Oxidized::Hook
         logger.debug "Authenticating using ssh keys as '#{git_user}'"
         rugged_sshkey(git_user: git_user, privkey: cfg.privatekey, pubkey: pubkey)
       elsif cfg.has_key?('remote_repo') &&
+            cfg.remote_repo.respond_to?(:has_key?) &&
             cfg.remote_repo.has_key?(node.group) &&
             cfg.remote_repo[node.group].has_key?('privatekey')
         pubkey = cfg.remote_repo[node.group].has_key?('publickey') ? cfg.remote_repo[node.group].publickey : nil

data/lib/oxidized/hook.rb

@@ -14,11 +14,14 @@ module Oxidized
       end
     end
 
-    # HookContext is passed to each hook. It can contain anything related to the
-    # event in question. At least it contains the event name
-    # The argument keyword_init: true is needed for ruby < 3.2 and can be
-    # dropped with the support of ruby 3.1
-    HookContext = Struct.new(:event, :node, :job, :commitref, keyword_init: true)
+    # HookContext is passed to each hook. It always carries the event name.
+    # The keyword_init: true argument forces keyword-argument initialization.
+    HookContext = Struct.new(
+      :event, :node, :job, :commitref,
+      :node_raw,   # raw source record: JSON hash, SQL row hash, CSV field array
+      :context,    # self from call site, to access methods/bindings of call site
+      keyword_init: true
+    )
 
     # RegisteredHook is a container for a Hook instance
     RegisteredHook = Struct.new(:name, :hook)
@@ -28,6 +31,7 @@ module Oxidized
       node_fail
       post_store
       nodes_done
+      source_node_transform
     ].freeze
     attr_reader :registered_hooks
 
@@ -54,16 +58,60 @@ module Oxidized
       logger.debug "Hook #{name.inspect} registered #{hook.class} for event #{event.inspect}"
     end
 
+    # --- Transform events ---
+
+    # Runs source_node_transform hooks in sequence, passing the return value of
+    # each hook as node to the next. Returns the final node, or nil
+    # to signal that the node should be excluded.
+    def source_node_transform(node:, node_raw:, context:)
+      ctx = HookContext.new(
+        event:    :source_node_transform,
+        node:     node,
+        node_raw: node_raw,
+        context:  context
+      )
+      @registered_hooks[:source_node_transform].each do |r_hook|
+        ctx.node = r_hook.hook.run_hook(ctx)
+      rescue StandardError => e
+        logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
+                     "(#{e.inspect}) for event :source_node_transform"
+      end
+      ctx.node
+    end
+
+    # --- Fire-and-forget events ---
+
+    def node_success(node:, job: nil)
+      handle(:node_success, node: node, job: job)
+    end
+
+    def node_fail(node:, job: nil)
+      handle(:node_fail, node: node, job: job)
+    end
+
+    def post_store(node:, job: nil, commitref: nil)
+      handle(:post_store, node: node, job: job, commitref: commitref)
+    end
+
+    def nodes_done
+      handle(:nodes_done)
+    end
+
+    private
+
+    # Shared implementation for fire-and-forget events: runs all registered
+    # hooks for the event, ignores return values, logs errors.
     def handle(event, ctx_params = {})
-      ctx = HookContext.new ctx_params
-      ctx.event = event
+      ctx = HookContext.new(event: event, **ctx_params)
 
       @registered_hooks[event].each do |r_hook|
-        r_hook.hook.run_hook ctx
+        r_hook.hook.run_hook(ctx)
       rescue StandardError => e
         logger.error "Hook #{r_hook.name} (#{r_hook.hook}) failed " \
                      "(#{e.inspect}) for event #{event.inspect}"
       end
+
+      nil
     end
   end
 

data/lib/oxidized/input/exec.rb

@@ -1,9 +1,5 @@
 module Oxidized
-  require "oxidized/input/cli"
-
   class Exec < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
       @log = File.open(Oxidized::Config::LOG + "/#{@node.ip}-exec", "w") if Oxidized.config.input.debug?

data/lib/oxidized/input/ftp.rb

@@ -1,20 +1,7 @@
 module Oxidized
   require 'net/ftp'
   require 'timeout'
-  require_relative 'cli'
-
   class FTP < Input
-    RESCUE_FAIL = {
-      debug: [
-        # Net::SSH::Disconnect,
-      ],
-      warn:  [
-        # RuntimeError,
-        # Net::SSH::AuthenticationFailed,
-      ]
-    }.freeze
-    include Input::CLI
-
     def connect(node) # rubocop:disable Naming/PredicateMethod
       @node = node
       @node.model.cfg['ftp'].each { |cb| instance_exec(&cb) }

data/lib/oxidized/input/http.rb

@@ -1,12 +1,9 @@
 module Oxidized
-  require "oxidized/input/cli"
   require "net/http"
   require "json"
   require "net/http/digest_auth"
 
   class HTTP < Input
-    include Input::CLI
-
     def connect(node)
       @node = node
       @secure = false
@@ -48,39 +45,67 @@ module Oxidized
     private
 
     def get_http(path)
+      res = perform_http_request(path, method: :get)
+      res.body
+    end
+
+    def post_http(path, body = nil, extra_headers = {})
+      res = perform_http_request(path, method: :post, body: body, extra_headers: extra_headers)
+      res.body
+    end
+
+    def perform_http_request(path, method: :get, body: nil, extra_headers: {})
       uri = get_uri(path)
+      http_method = method.to_s.upcase
 
-      logger.debug "Making request to: #{uri}"
+      logger.debug "Making #{http_method} request to: #{uri}"
 
       ssl_verify = Oxidized.config.input.http.ssl_verify? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
 
-      res = make_request(uri, ssl_verify)
+      res = make_request(uri, ssl_verify, extra_headers, method: method, body: body)
 
       if res.code == '401' && res['www-authenticate']&.include?('Digest')
         uri.user = @username
         uri.password = URI.encode_www_form_component(@password)
         logger.debug "Server requires Digest authentication"
-        auth = Net::HTTP::DigestAuth.new.auth_header(uri, res['www-authenticate'], 'GET')
 
-        res = make_request(uri, ssl_verify, 'Authorization' => auth)
-      elsif @username && @password
+        auth = Net::HTTP::DigestAuth.new.auth_header(uri, res['www-authenticate'], http_method)
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => auth),
+                           method: method, body: body)
+
+      elsif @username && @password && !authorization_header_present?(extra_headers)
         logger.debug "Falling back to Basic authentication"
-        res = make_request(uri, ssl_verify, 'Authorization' => basic_auth_header)
+        res = make_request(uri, ssl_verify, extra_headers.merge('Authorization' => basic_auth_header),
+                           method: method, body: body)
       end
 
       logger.debug "Response code: #{res.code}"
-      res.body
+      res
     end
 
-    def make_request(uri, ssl_verify, extra_headers = {})
+    def make_request(uri, ssl_verify, extra_headers = {}, method: :get, body: nil)
       Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https", verify_mode: ssl_verify) do |http|
-        req = Net::HTTP::Get.new(uri)
+        req_class = if method == :get
+                      Net::HTTP::Get
+                    elsif method == :post
+                      Net::HTTP::Post
+                    else
+                      raise Oxidized::OxidizedError, "Unsupported HTTP method: #{method.inspect}. " \
+                                                     "Only :get and :post are supported"
+                    end
+        req = req_class.new(uri)
         @headers.merge(extra_headers).each { |header, value| req.add_field(header, value) }
-        logger.debug "Sending request with headers: #{@headers.merge(extra_headers)}"
+        req.body = body if body
+
+        logger.debug "Sending #{method.to_s.upcase} request with headers: #{@headers.merge(extra_headers)}"
         http.request(req)
       end
     end
 
+    def authorization_header_present?(headers)
+      headers.keys.any? { |key| key.to_s.downcase == 'authorization' }
+    end
+
     def basic_auth_header
       "Basic " + ["#{@username}:#{@password}"].pack('m').delete("\r\n")
     end

data/lib/oxidized/input/input.rb

@@ -1,24 +1,44 @@
+require_relative 'cli'
+
 module Oxidized
   class PromptUndetect < OxidizedError; end
 
   class Input
     include SemanticLogger::Loggable
     include Oxidized::Config::Vars
+    include Oxidized::Input::CLI
 
     RESCUE_FAIL = {
-      debug: [
-        Errno::ECONNREFUSED
-      ],
-      warn:  [
-        IOError,
-        PromptUndetect,
-        Timeout::Error,
-        Errno::ECONNRESET,
-        Errno::EHOSTUNREACH,
-        Errno::ENETUNREACH,
-        Errno::EPIPE,
-        Errno::ETIMEDOUT
-      ]
+      Errno::ECONNREFUSED => :debug,
+      IOError             => :warn,
+      PromptUndetect      => :warn,
+      Timeout::Error      => :warn,
+      Errno::ECONNRESET   => :warn,
+      Errno::EHOSTUNREACH => :warn,
+      Errno::ENETUNREACH  => :warn,
+      Errno::EPIPE        => :warn,
+      Errno::ETIMEDOUT    => :warn
     }.freeze
+
+    # Returns a hash mapping exception classes to their log level
+    def self.rescue_fail
+      RESCUE_FAIL.dup
+    end
+
+    def self.config_name
+      name.split('::').last.downcase
+    end
+
+    def self.to_sym
+      config_name.to_sym
+    end
+
+    def config_name
+      self.class.config_name
+    end
+
+    def to_sym
+      self.class.to_sym
+    end
   end
 end

data/lib/oxidized/input/scp.rb

@@ -1,54 +1,20 @@
 module Oxidized
   require 'net/ssh'
-  require 'net/scp'
+  begin
+    require 'net/scp'
+  rescue LoadError
+    raise OxidizedError, 'net/scp not found: sudo gem install net-scp'
+  end
   require 'timeout'
-  require_relative 'cli'
+  require_relative 'sshbase'
 
-  class SCP < Input
+  class SCP < SSHBase
     RESCUE_FAIL = {
-      debug: [
-        Net::SSH::Disconnect,
-        Net::SSH::ConnectionTimeout
-      ],
-      warn:  [
-        Net::SCP::Error,
-        Net::SSH::HostKeyUnknown,
-        Net::SSH::AuthenticationFailed,
-        Timeout::Error
-      ]
+      Net::SCP::Error => :warn
     }.freeze
-    include Input::CLI
-
-    def connect(node) # rubocop:disable Naming/PredicateMethod
-      @node = node
-      @node.model.cfg['scp'].each { |cb| instance_exec(&cb) }
-      @log = File.open(Oxidized::Config::LOG + "/#{@node.ip}-scp", 'w') if Oxidized.config.input.debug?
-      @ssh = Net::SSH.start(@node.ip, @node.auth[:username], make_ssh_opts)
-      connected?
-    end
-
-    def make_ssh_opts
-      secure = Oxidized.config.input.scp.secure?
-      ssh_opts = {
-        number_of_password_prompts:      0,
-        verify_host_key:                 secure ? :always : :never,
-        append_all_supported_algorithms: true,
-        password:                        @node.auth[:password],
-        timeout:                         @node.timeout,
-        port:                            (vars(:ssh_port) || 22).to_i,
-        forward_agent:                   false
-      }
 
-      # Use our logger for Net::SSH
-      ssh_logger = SemanticLogger[Net::SSH]
-      ssh_logger.level = Oxidized.config.input.debug? ? :debug : :fatal
-      ssh_opts[:logger] = ssh_logger
-
-      ssh_opts
-    end
-
-    def connected?
-      @ssh && (not @ssh.closed?)
+    def self.rescue_fail
+      super.merge(RESCUE_FAIL)
     end
 
     def cmd(file)
@@ -57,25 +23,5 @@ module Oxidized
         @ssh.scp.download!(file)
       end
     end
-
-    def send(my_proc)
-      my_proc.call
-    end
-
-    def output
-      ""
-    end
-
-    private
-
-    def disconnect
-      Timeout.timeout(@node.timeout) do
-        @ssh.close
-      end
-    rescue Timeout::Error
-      logger.debug "#{@node.name} timed out while disconnecting"
-    ensure
-      @log.close if Oxidized.config.input.debug?
-    end
   end
 end