oxidized 0.29.1 → 0.37.0

data/lib/oxidized/source/csv.rb

@@ -1,60 +1,60 @@
 module Oxidized
-  class CSV < Source
-    def initialize
-      @cfg = Oxidized.config.source.csv
-      super
-    end
-
-    def setup
-      if @cfg.empty?
-        Oxidized.asetus.user.source.csv.file      = File.join(Config::Root, 'router.db')
-        Oxidized.asetus.user.source.csv.delimiter = /:/
-        Oxidized.asetus.user.source.csv.map.name  = 0
-        Oxidized.asetus.user.source.csv.map.model = 1
-        Oxidized.asetus.user.source.csv.gpg       = false
-        Oxidized.asetus.save :user
-        raise NoConfig, 'no source csv config, edit ~/.config/oxidized/config'
+  module Source
+    class CSV < Source
+      def initialize
+        @cfg = Oxidized.config.source.csv
+        super
       end
-      require 'gpgme' if @cfg.gpg?
-    end
-
-    def load(_node_want = nil)
-      nodes = []
-      open_file.each_line do |line|
-        next if line =~ /^\s*#/
-
-        data = line.chomp.split(@cfg.delimiter, -1)
-        next if data.empty?
 
-        # map node parameters
-        keys = {}
-        @cfg.map.each do |key, position|
-          keys[key.to_sym] = node_var_interpolate data[position]
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.csv.file      = File.join(Config::ROOT, 'router.db')
+          Oxidized.asetus.user.source.csv.delimiter = /:/
+          Oxidized.asetus.user.source.csv.map.name  = 0
+          Oxidized.asetus.user.source.csv.map.model = 1
+          Oxidized.asetus.user.source.csv.gpg       = false
+          Oxidized.asetus.save :user
+          raise NoConfig, "no source csv config, edit #{Oxidized::Config.configfile}"
         end
-        keys[:model] = map_model keys[:model] if keys.has_key? :model
-        keys[:group] = map_group keys[:group] if keys.has_key? :group
+        require 'gpgme' if @cfg.gpg?
 
-        # map node specific vars
-        vars = {}
-        @cfg.vars_map.each do |key, position|
-          vars[key.to_sym] = node_var_interpolate data[position]
-        end
-        keys[:vars] = vars unless vars.empty?
+        # map.name is mandatory
+        return if @cfg.map.has_key?('name')
 
-        nodes << keys
+        raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
       end
-      nodes
-    end
 
-    private
-
-    def open_file
-      file = File.expand_path(@cfg.file)
-      if @cfg.gpg?
-        crypto = GPGME::Crypto.new password: @cfg.gpg_password
-        crypto.decrypt(File.open(file)).to_s
-      else
-        File.open(file)
+      def load(_node_want = nil)
+        nodes = []
+        open_file.each_line do |line|
+          next if line =~ /^\s*#/
+
+          data = line.chomp.split(@cfg.delimiter, -1)
+          next if data.empty?
+
+          # map node parameters
+          keys = {}
+          @cfg.map.each do |key, position|
+            keys[key.to_sym] = node_var_interpolate data[position]
+          end
+          keys[:model] = map_model keys[:model] if keys.has_key? :model
+          keys[:group] = map_group keys[:group] if keys.has_key? :group
+
+          # map node specific vars
+          vars = {}
+          @cfg.vars_map.each do |key, position|
+            vars[key.to_s] = node_var_interpolate data[position]
+          end
+          keys[:vars] = vars unless vars.empty?
+
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: data,
+                                                      context:  self)
+          next if keys.nil?
+
+          nodes << keys
+        end
+        nodes
       end
     end
   end

data/lib/oxidized/source/http.rb

@@ -1,81 +1,94 @@
 module Oxidized
-  class HTTP < Source
-    def initialize
-      @cfg = Oxidized.config.source.http
-      super
-    end
+  module Source
+    require "oxidized/source/jsonfile"
+    class HTTP < JSONFile
+      def initialize
+        super
+        @cfg = Oxidized.config.source.http
+      end
 
-    def setup
-      return unless @cfg.url.empty?
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.http.url       = 'https://url/api'
+          Oxidized.asetus.user.source.http.map.name  = 'name'
+          Oxidized.asetus.user.source.http.map.model = 'model'
+          Oxidized.asetus.save :user
 
-      raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config'
-    end
-
-    require "net/http"
-    require "net/https"
-    require "uri"
-    require "json"
-
-    def load(node_want = nil)
-      nodes = []
-      data = JSON.parse(read_http(node_want))
-      data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
-      data.each do |node|
-        next if node.empty?
-
-        # map node parameters
-        keys = {}
-        @cfg.map.each do |key, want_position|
-          keys[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+          raise NoConfig, "No source http config, edit #{Oxidized::Config.configfile}"
         end
-        keys[:model] = map_model keys[:model] if keys.has_key? :model
-        keys[:group] = map_group keys[:group] if keys.has_key? :group
 
-        # map node specific vars
-        vars = {}
-        @cfg.vars_map.each do |key, want_position|
-          vars[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+        # check for mandatory attributes
+        if !@cfg.has_key?('url')
+          raise InvalidConfig, "url is a mandatory http source attribute, edit #{Oxidized::Config.configfile}"
+        elsif !@cfg.map.has_key?('name')
+          raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
         end
-        keys[:vars] = vars unless vars.empty?
-
-        nodes << keys
       end
-      nodes
-    end
 
-    private
+      require "net/http"
+      require "net/https"
+      require "uri"
+      require "json"
 
-    def string_navigate(object, wants)
-      wants = wants.split(".").map do |want|
-        head, match, _tail = want.partition(/\[\d+\]/)
-        match.empty? ? head : [head, match[1..-2].to_i]
-      end
-      wants.flatten.each do |want|
-        object = object[want] if object.respond_to? :each
+      def load(node_want = nil)
+        uri = URI.parse(@cfg.url)
+        data = JSON.parse(read_http(uri, node_want))
+        node_data = data
+        node_data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+        node_data = pagination(data, node_want) if @cfg.pagination?
+
+        transform_json(node_data)
       end
-      object
-    end
 
-    def read_http(node_want)
-      uri = URI.parse(@cfg.url)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if uri.scheme == 'https'
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
+      private
+
+      def pagination(data, node_want)
+        node_data = []
+        unless @cfg.pagination_key_name?
+          raise Oxidized::OxidizedError,
+                "if using pagination, 'pagination_key_name' setting must be set"
+        end
+
+        next_key = @cfg.pagination_key_name
+        # Process the initial batch first
+        node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+
+        # Now loop only for subsequent pages
+        loop do
+          # Check for the next URL *before* trying to fetch/add
+          break if data[next_key].nil?
 
-      # Add read_timeout to handle case of big list of nodes (default value is 60 seconds)
-      http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout"
+          # Fetch the next page
+          new_uri = URI.parse(data[next_key]) # Safe to parse now
+          data = JSON.parse(read_http(new_uri, node_want))
+
+          # Add nodes from the *newly fetched* batch
+          node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+        end
 
-      # map headers
-      headers = {}
-      @cfg.headers.each do |header, value|
-        headers[header] = value
+        node_data
       end
 
-      req_uri = uri.request_uri
-      req_uri = "#{req_uri}/#{node_want}" if node_want
-      request = Net::HTTP::Get.new(req_uri, headers)
-      request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
-      http.request(request).body
+      def read_http(uri, node_want)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true if uri.scheme == 'https'
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
+
+        # Add read_timeout to handle case of big list of nodes (default value is 60 seconds)
+        http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout"
+
+        # map headers
+        headers = {}
+        @cfg.headers.each do |header, value|
+          headers[header] = value
+        end
+
+        req_uri = uri.request_uri
+        req_uri = "#{req_uri}/#{node_want}" if node_want
+        request = Net::HTTP::Get.new(req_uri, headers)
+        request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
+        http.request(request).body
+      end
     end
   end
 end

data/lib/oxidized/source/jsonfile.rb

@@ -0,0 +1,68 @@
+module Oxidized
+  module Source
+    class JSONFile < Source
+      require "json"
+      def initialize
+        @cfg = Oxidized.config.source.jsonfile
+        super
+      end
+
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.jsonfile.file      = File.join(Oxidized::Config::ROOT,
+                                                                     'router.json')
+          Oxidized.asetus.user.source.jsonfile.map.name  = "name"
+          Oxidized.asetus.user.source.jsonfile.map.model = "model"
+          Oxidized.asetus.user.source.jsonfile.gpg       = false
+          Oxidized.asetus.save :user
+          raise NoConfig, "No source json config, edit #{Oxidized::Config.configfile}"
+        end
+        require 'gpgme' if @cfg.gpg?
+
+        # map.name is mandatory
+        return if @cfg.map.has_key?('name')
+
+        raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
+      end
+
+      def load(*)
+        data = JSON.parse(open_file.read)
+        data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+
+        transform_json(data)
+      end
+
+      private
+
+      def transform_json(data)
+        nodes = []
+        data.each do |node|
+          next if node.empty?
+
+          # map node parameters
+          keys = {}
+          @cfg.map.each do |key, want_position|
+            keys[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position)
+          end
+          keys[:model] = map_model keys[:model] if keys.has_key? :model
+          keys[:group] = map_group keys[:group] if keys.has_key? :group
+
+          # map node specific vars
+          vars = {}
+          @cfg.vars_map.each do |key, want_position|
+            vars[key.to_s] = node_var_interpolate string_navigate_object(node, want_position)
+          end
+          keys[:vars] = vars unless vars.empty?
+
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node,
+                                                      context:  self)
+          next if keys.nil?
+
+          nodes << keys
+        end
+        nodes
+      end
+    end
+  end
+end

data/lib/oxidized/source/source.rb

@@ -1,26 +1,81 @@
 module Oxidized
-  class Source
-    class NoConfig < OxidizedError; end
+  module Source
+    class Source
+      class NoConfig < OxidizedError; end
 
-    def initialize
-      @model_map = (Oxidized.config.model_map || {})
-      @group_map = (Oxidized.config.group_map || {})
-    end
+      def initialize
+        @model_map = Oxidized.config.model_map || {}
+        @group_map = Oxidized.config.group_map || {}
+      end
 
-    def map_model(model)
-      @model_map.has_key?(model) ? @model_map[model] : model
-    end
+      # common code of #map_model and #map_group
+      def map_value(map_hash, original_value)
+        map_hash.each do |key, new_value|
+          mthd = key.instance_of?(Regexp) ? :match : :eql?
+          return new_value if original_value.send(mthd, key)
+        end
+        original_value
+      end
 
-    def map_group(group)
-      @group_map.has_key?(group) ? @group_map[group] : group
-    end
+      # search a match for model in the configuration and returns it.
+      # If no match is found, return model
+      #
+      # model can be matched against a string or a regexp:
+      #
+      # model_map:
+      #   cisco: ios
+      #   juniper: junos
+      #   !ruby/regexp /procurve/: procurve
+      def map_model(model)
+        map_value(@model_map, model)
+      end
+
+      # search a match for group in the configuration and returns it.
+      # If no match is found, return group
+      #
+      # group can be matched against a string or a regexp:
+      #
+      # group_map:
+      #   alias1: groupA
+      #   alias2: groupA
+      #   alias3: groupB
+      #   alias4: groupB
+      #   !ruby/regexp /specialgroup/: groupS
+      #   aliasN: groupZ
+      def map_group(group)
+        map_value(@group_map, group)
+      end
+
+      def node_var_interpolate(var)
+        case var
+        when "nil"   then nil
+        when "false" then false
+        when "true"  then true
+        else var
+        end
+      end
+
+      private
+
+      def open_file
+        file = File.expand_path(@cfg.file)
+        if @cfg.gpg?
+          crypto = GPGME::Crypto.new password: @cfg.gpg_password
+          crypto.decrypt(File.open(file)).to_s
+        else
+          File.open(file)
+        end
+      end
 
-    def node_var_interpolate(var)
-      case var
-      when "nil"   then nil
-      when "false" then false
-      when "true"  then true
-      else var
+      def string_navigate_object(object, wants)
+        wants = wants.split(".").map do |want|
+          head, match, _tail = want.partition(/\[\d+\]/)
+          match.empty? ? head : [head, match[1..-2].to_i]
+        end
+        wants.flatten.each do |want|
+          object = object[want] if object.respond_to? :each
+        end
+        object
       end
     end
   end

data/lib/oxidized/source/sql.rb

@@ -1,75 +1,87 @@
 module Oxidized
-  class SQL < Source
-    begin
-      require 'sequel'
-    rescue LoadError
-      raise OxidizedError, 'sequel not found: sudo gem install sequel'
-    end
+  module Source
+    class SQL < Source
+      begin
+        require 'sequel'
+      rescue LoadError
+        raise OxidizedError, 'sequel not found: sudo gem install sequel'
+      end
 
-    def setup
-      return unless @cfg.empty?
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.sql.adapter   = 'sqlite'
+          Oxidized.asetus.user.source.sql.database  = File.join(Config::ROOT, 'sqlite.db')
+          Oxidized.asetus.user.source.sql.table     = 'devices'
+          Oxidized.asetus.user.source.sql.map.name  = 'name'
+          Oxidized.asetus.user.source.sql.map.model = 'rancid'
+          Oxidized.asetus.save :user
+          raise NoConfig, "No source sql config, edit #{Oxidized::Config.configfile}"
+        end
 
-      Oxidized.asetus.user.source.sql.adapter   = 'sqlite'
-      Oxidized.asetus.user.source.sql.database  = File.join(Config::Root, 'sqlite.db')
-      Oxidized.asetus.user.source.sql.table     = 'devices'
-      Oxidized.asetus.user.source.sql.map.name  = 'name'
-      Oxidized.asetus.user.source.sql.map.model = 'rancid'
-      Oxidized.asetus.save :user
-      raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
-    end
+        # map.name is mandatory
+        return if @cfg.map.has_key?('name')
 
-    def load(node_want = nil)
-      nodes = []
-      db = connect
-      query = db[@cfg.table.to_sym]
-      query = query.with_sql(@cfg.query) if @cfg.query?
+        raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
+      end
 
-      query = query.where(@cfg.map.name.to_sym => node_want) if node_want
+      def load(node_want = nil)
+        nodes = []
+        db = connect
+        query = db[@cfg.table.to_sym]
+        query = query.with_sql(@cfg.query) if @cfg.query?
 
-      query.each do |node|
-        # map node parameters
-        keys = {}
-        @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
-        keys[:model] = map_model keys[:model] if keys.has_key? :model
-        keys[:group] = map_group keys[:group] if keys.has_key? :group
+        query = query.where(@cfg.map.name.to_sym => node_want) if node_want
 
-        # map node specific vars
-        vars = {}
-        @cfg.vars_map.each do |key, sql_column|
-          vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym]
-        end
-        keys[:vars] = vars unless vars.empty?
+        query.each do |node|
+          # map node parameters
+          keys = {}
+          @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
+          keys[:model] = map_model keys[:model] if keys.has_key? :model
+          keys[:group] = map_group keys[:group] if keys.has_key? :group
+
+          # map node specific vars
+          vars = {}
+          @cfg.vars_map.each do |key, sql_column|
+            vars[key.to_s] = node_var_interpolate node[sql_column.to_sym]
+          end
+          keys[:vars] = vars unless vars.empty?
 
-        nodes << keys
+          keys = Oxidized.hooks.source_node_transform(node:     keys,
+                                                      node_raw: node.to_hash,
+                                                      context:  self)
+          next if keys.nil?
+
+          nodes << keys
+        end
+        db.disconnect
+        nodes
       end
-      db.disconnect
-      nodes
-    end
 
-    private
+      private
 
-    def initialize
-      super
-      @cfg = Oxidized.config.source.sql
-    end
+      def initialize
+        super
+        @cfg = Oxidized.config.source.sql
+      end
 
-    def connect
-      options = {
-        adapter:  @cfg.adapter,
-        host:     @cfg.host?,
-        user:     @cfg.user?,
-        password: @cfg.password?,
-        database: @cfg.database,
-        ssl_mode: @cfg.ssl_mode?
-      }
-      if @cfg.with_ssl?
-        options.merge!(sslca:   @cfg.ssl_ca?,
-                       sslcert: @cfg.ssl_cert?,
-                       sslkey:  @cfg.ssl_key?)
+      def connect
+        options = {
+          adapter:  @cfg.adapter,
+          host:     @cfg.host?,
+          user:     @cfg.user?,
+          password: @cfg.password?,
+          database: @cfg.database,
+          ssl_mode: @cfg.ssl_mode?
+        }
+        if @cfg.with_ssl?
+          options.merge!(sslca:   @cfg.ssl_ca?,
+                         sslcert: @cfg.ssl_cert?,
+                         sslkey:  @cfg.ssl_key?)
+        end
+        Sequel.connect(options)
+      rescue Sequel::AdapterNotFound => e
+        raise OxidizedError, "SQL adapter gem not installed: " + e.message
       end
-      Sequel.connect(options)
-    rescue Sequel::AdapterNotFound => error
-      raise OxidizedError, "SQL adapter gem not installed: " + error.message
     end
   end
 end

data/lib/oxidized/version.rb

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 module Oxidized
-  VERSION = '0.29.1'.freeze
-  VERSION_FULL = '0.29.1'.freeze
+  VERSION = '0.37.0'
+  VERSION_FULL = '0.37.0'
   def self.version_set
     version_full = %x(git describe --tags).chop rescue ""
     version      = %x(git describe --tags --abbrev=0).chop rescue ""
@@ -12,8 +14,8 @@ module Oxidized
     const_set(:VERSION, version)
     const_set(:VERSION_FULL, version_full)
     file = File.readlines(__FILE__)
-    file[1] = "  VERSION = '%s'.freeze\n" % VERSION
-    file[2] = "  VERSION_FULL = '%s'.freeze\n" % VERSION_FULL
+    file[3] = "  VERSION = '%s'\n" % VERSION
+    file[4] = "  VERSION_FULL = '%s'\n" % VERSION_FULL
     File.write(__FILE__, file.join)
   end
 end