oxidized 0.30.1 → 0.32.0

data/lib/oxidized/source/http.rb

@@ -1,102 +1,84 @@
 module Oxidized
-  class HTTP < Source
-    def initialize
-      @cfg = Oxidized.config.source.http
-      super
-    end
-
-    def setup
-      Oxidized.setup_logger
-      return unless @cfg.url.empty?
-
-      raise NoConfig, 'no source http url config, edit ~/.config/oxidized/config'
-    end
-
-    require "net/http"
-    require "net/https"
-    require "uri"
-    require "json"
-
-    def load(node_want = nil)
-      nodes = []
-      uri = URI.parse(@cfg.url)
-      data = JSON.parse(read_http(uri, node_want))
-      node_data = data
-      node_data = string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
-      node_data = pagination(data, node_want) if @cfg.pagination?
+  module Source
+    require "oxidized/source/jsonfile"
+    class HTTP < JSONFile
+      def initialize
+        super
+        @cfg = Oxidized.config.source.http
+      end
 
-      # at this point we have all the nodes; pagination or not
-      node_data.each do |node|
-        next if node.empty?
+      def setup
+        Oxidized.setup_logger
+        if @cfg.empty?
+          Oxidized.asetus.user.source.http.url       = 'https://url/api'
+          Oxidized.asetus.user.source.http.map.name  = 'name'
+          Oxidized.asetus.user.source.http.map.model = 'model'
+          Oxidized.asetus.save :user
 
-        # map node parameters
-        keys = {}
-        @cfg.map.each do |key, want_position|
-          keys[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+          raise NoConfig, "No source http config, edit #{Oxidized::Config.configfile}"
         end
-        keys[:model] = map_model keys[:model] if keys.has_key? :model
-        keys[:group] = map_group keys[:group] if keys.has_key? :group
 
-        # map node specific vars
-        vars = {}
-        @cfg.vars_map.each do |key, want_position|
-          vars[key.to_sym] = node_var_interpolate string_navigate(node, want_position)
+        # check for mandatory attributes
+        if !@cfg.has_key?('url')
+          raise InvalidConfig, "url is a mandatory http source attribute, edit #{Oxidized::Config.configfile}"
+        elsif !@cfg.map.has_key?('name')
+          raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
         end
-        keys[:vars] = vars unless vars.empty?
-
-        nodes << keys
       end
-      nodes
-    end
 
-    private
+      require "net/http"
+      require "net/https"
+      require "uri"
+      require "json"
 
-    def string_navigate(object, wants)
-      wants = wants.split(".").map do |want|
-        head, match, _tail = want.partition(/\[\d+\]/)
-        match.empty? ? head : [head, match[1..-2].to_i]
-      end
-      wants.flatten.each do |want|
-        object = object[want] if object.respond_to? :each
+      def load(node_want = nil)
+        uri = URI.parse(@cfg.url)
+        data = JSON.parse(read_http(uri, node_want))
+        node_data = data
+        node_data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+        node_data = pagination(data, node_want) if @cfg.pagination?
+
+        transform_json(node_data)
       end
-      object
-    end
 
-    def pagination(data, node_want)
-      node_data = []
-      raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name?
+      private
 
-      next_key = @cfg.pagination_key_name
-      loop do
-        node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
-        break if data[next_key].nil?
+      def pagination(data, node_want)
+        node_data = []
+        raise Oxidized::OxidizedError, "if using pagination, 'pagination_key_name' setting must be set" unless @cfg.pagination_key_name?
 
-        new_uri = URI.parse(data[next_key]) if data.has_key?(next_key)
-        data = JSON.parse(read_http(new_uri, node_want))
-        node_data += string_navigate(data, @cfg.hosts_location) if @cfg.hosts_location?
+        next_key = @cfg.pagination_key_name
+        loop do
+          node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+          break if data[next_key].nil?
+
+          new_uri = URI.parse(data[next_key]) if data.has_key?(next_key)
+          data = JSON.parse(read_http(new_uri, node_want))
+          node_data += string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+        end
+        node_data
       end
-      node_data
-    end
 
-    def read_http(uri, node_want)
-      http = Net::HTTP.new(uri.host, uri.port)
-      http.use_ssl = true if uri.scheme == 'https'
-      http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
+      def read_http(uri, node_want)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true if uri.scheme == 'https'
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless @cfg.secure
 
-      # Add read_timeout to handle case of big list of nodes (default value is 60 seconds)
-      http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout"
+        # Add read_timeout to handle case of big list of nodes (default value is 60 seconds)
+        http.read_timeout = Integer(@cfg.read_timeout) if @cfg.has_key? "read_timeout"
 
-      # map headers
-      headers = {}
-      @cfg.headers.each do |header, value|
-        headers[header] = value
-      end
+        # map headers
+        headers = {}
+        @cfg.headers.each do |header, value|
+          headers[header] = value
+        end
 
-      req_uri = uri.request_uri
-      req_uri = "#{req_uri}/#{node_want}" if node_want
-      request = Net::HTTP::Get.new(req_uri, headers)
-      request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
-      http.request(request).body
+        req_uri = uri.request_uri
+        req_uri = "#{req_uri}/#{node_want}" if node_want
+        request = Net::HTTP::Get.new(req_uri, headers)
+        request.basic_auth(@cfg.user, @cfg.pass) if @cfg.user? && @cfg.pass?
+        http.request(request).body
+      end
     end
   end
 end

data/lib/oxidized/source/jsonfile.rb

@@ -0,0 +1,63 @@
+module Oxidized
+  module Source
+    class JSONFile < Source
+      require "json"
+      def initialize
+        @cfg = Oxidized.config.source.jsonfile
+        super
+      end
+
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.jsonfile.file      = File.join(Oxidized::Config::ROOT,
+                                                                     'router.json')
+          Oxidized.asetus.user.source.jsonfile.map.name  = "name"
+          Oxidized.asetus.user.source.jsonfile.map.model = "model"
+          Oxidized.asetus.user.source.jsonfile.gpg       = false
+          Oxidized.asetus.save :user
+          raise NoConfig, "No source json config, edit #{Oxidized::Config.configfile}"
+        end
+        require 'gpgme' if @cfg.gpg?
+
+        # map.name is mandatory
+        return if @cfg.map.has_key?('name')
+
+        raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
+      end
+
+      def load(*)
+        data = JSON.parse(open_file.read)
+        data = string_navigate_object(data, @cfg.hosts_location) if @cfg.hosts_location?
+
+        transform_json(data)
+      end
+
+      private
+
+      def transform_json(data)
+        nodes = []
+        data.each do |node|
+          next if node.empty?
+
+          # map node parameters
+          keys = {}
+          @cfg.map.each do |key, want_position|
+            keys[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position)
+          end
+          keys[:model] = map_model keys[:model] if keys.has_key? :model
+          keys[:group] = map_group keys[:group] if keys.has_key? :group
+
+          # map node specific vars
+          vars = {}
+          @cfg.vars_map.each do |key, want_position|
+            vars[key.to_sym] = node_var_interpolate string_navigate_object(node, want_position)
+          end
+          keys[:vars] = vars unless vars.empty?
+
+          nodes << keys
+        end
+        nodes
+      end
+    end
+  end
+end

data/lib/oxidized/source/source.rb

@@ -1,26 +1,81 @@
 module Oxidized
-  class Source
-    class NoConfig < OxidizedError; end
+  module Source
+    class Source
+      class NoConfig < OxidizedError; end
 
-    def initialize
-      @model_map = Oxidized.config.model_map || {}
-      @group_map = Oxidized.config.group_map || {}
-    end
+      def initialize
+        @model_map = Oxidized.config.model_map || {}
+        @group_map = Oxidized.config.group_map || {}
+      end
 
-    def map_model(model)
-      @model_map.has_key?(model) ? @model_map[model] : model
-    end
+      # common code of #map_model and #map_group
+      def map_value(map_hash, original_value)
+        map_hash.each do |key, new_value|
+          mthd = key.instance_of?(Regexp) ? :match : :eql?
+          return new_value if original_value.send(mthd, key)
+        end
+        original_value
+      end
 
-    def map_group(group)
-      @group_map.has_key?(group) ? @group_map[group] : group
-    end
+      # search a match for model in the configuration and returns it.
+      # If no match is found, return model
+      #
+      # model can be matched against a string or a regexp:
+      #
+      # model_map:
+      #   cisco: ios
+      #   juniper: junos
+      #   !ruby/regexp /procurve/: procurve
+      def map_model(model)
+        map_value(@model_map, model)
+      end
+
+      # search a match for group in the configuration and returns it.
+      # If no match is found, return group
+      #
+      # group can be matched against a string or a regexp:
+      #
+      # group_map:
+      #   alias1: groupA
+      #   alias2: groupA
+      #   alias3: groupB
+      #   alias4: groupB
+      #   !ruby/regexp /specialgroup/: groupS
+      #   aliasN: groupZ
+      def map_group(group)
+        map_value(@group_map, group)
+      end
+
+      def node_var_interpolate(var)
+        case var
+        when "nil"   then nil
+        when "false" then false
+        when "true"  then true
+        else var
+        end
+      end
+
+      private
+
+      def open_file
+        file = File.expand_path(@cfg.file)
+        if @cfg.gpg?
+          crypto = GPGME::Crypto.new password: @cfg.gpg_password
+          crypto.decrypt(File.open(file)).to_s
+        else
+          File.open(file)
+        end
+      end
 
-    def node_var_interpolate(var)
-      case var
-      when "nil"   then nil
-      when "false" then false
-      when "true"  then true
-      else var
+      def string_navigate_object(object, wants)
+        wants = wants.split(".").map do |want|
+          head, match, _tail = want.partition(/\[\d+\]/)
+          match.empty? ? head : [head, match[1..-2].to_i]
+        end
+        wants.flatten.each do |want|
+          object = object[want] if object.respond_to? :each
+        end
+        object
       end
     end
   end

data/lib/oxidized/source/sql.rb

@@ -1,75 +1,82 @@
 module Oxidized
-  class SQL < Source
-    begin
-      require 'sequel'
-    rescue LoadError
-      raise OxidizedError, 'sequel not found: sudo gem install sequel'
-    end
+  module Source
+    class SQL < Source
+      begin
+        require 'sequel'
+      rescue LoadError
+        raise OxidizedError, 'sequel not found: sudo gem install sequel'
+      end
 
-    def setup
-      return unless @cfg.empty?
+      def setup
+        if @cfg.empty?
+          Oxidized.asetus.user.source.sql.adapter   = 'sqlite'
+          Oxidized.asetus.user.source.sql.database  = File.join(Config::ROOT, 'sqlite.db')
+          Oxidized.asetus.user.source.sql.table     = 'devices'
+          Oxidized.asetus.user.source.sql.map.name  = 'name'
+          Oxidized.asetus.user.source.sql.map.model = 'rancid'
+          Oxidized.asetus.save :user
+          raise NoConfig, "No source sql config, edit #{Oxidized::Config.configfile}"
+        end
 
-      Oxidized.asetus.user.source.sql.adapter   = 'sqlite'
-      Oxidized.asetus.user.source.sql.database  = File.join(Config::ROOT, 'sqlite.db')
-      Oxidized.asetus.user.source.sql.table     = 'devices'
-      Oxidized.asetus.user.source.sql.map.name  = 'name'
-      Oxidized.asetus.user.source.sql.map.model = 'rancid'
-      Oxidized.asetus.save :user
-      raise NoConfig, 'no source sql config, edit ~/.config/oxidized/config'
-    end
+        # map.name is mandatory
+        return if @cfg.map.has_key?('name')
 
-    def load(node_want = nil)
-      nodes = []
-      db = connect
-      query = db[@cfg.table.to_sym]
-      query = query.with_sql(@cfg.query) if @cfg.query?
+        raise InvalidConfig, "map/name is a mandatory source attribute, edit #{Oxidized::Config.configfile}"
+      end
 
-      query = query.where(@cfg.map.name.to_sym => node_want) if node_want
+      def load(node_want = nil)
+        nodes = []
+        db = connect
+        query = db[@cfg.table.to_sym]
+        query = query.with_sql(@cfg.query) if @cfg.query?
 
-      query.each do |node|
-        # map node parameters
-        keys = {}
-        @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
-        keys[:model] = map_model keys[:model] if keys.has_key? :model
-        keys[:group] = map_group keys[:group] if keys.has_key? :group
+        query = query.where(@cfg.map.name.to_sym => node_want) if node_want
 
-        # map node specific vars
-        vars = {}
-        @cfg.vars_map.each do |key, sql_column|
-          vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym]
-        end
-        keys[:vars] = vars unless vars.empty?
+        query.each do |node|
+          # map node parameters
+          keys = {}
+          @cfg.map.each { |key, sql_column| keys[key.to_sym] = node_var_interpolate node[sql_column.to_sym] }
+          keys[:model] = map_model keys[:model] if keys.has_key? :model
+          keys[:group] = map_group keys[:group] if keys.has_key? :group
 
-        nodes << keys
+          # map node specific vars
+          vars = {}
+          @cfg.vars_map.each do |key, sql_column|
+            vars[key.to_sym] = node_var_interpolate node[sql_column.to_sym]
+          end
+          keys[:vars] = vars unless vars.empty?
+
+          nodes << keys
+        end
+        db.disconnect
+        nodes
       end
-      db.disconnect
-      nodes
-    end
 
-    private
+      private
 
-    def initialize
-      super
-      @cfg = Oxidized.config.source.sql
-    end
+      def initialize
+        super
+        @cfg = Oxidized.config.source.sql
+      end
 
-    def connect
-      options = {
-        adapter:  @cfg.adapter,
-        host:     @cfg.host?,
-        user:     @cfg.user?,
-        password: @cfg.password?,
-        database: @cfg.database,
-        ssl_mode: @cfg.ssl_mode?
-      }
-      if @cfg.with_ssl?
-        options.merge!(sslca:   @cfg.ssl_ca?,
-                       sslcert: @cfg.ssl_cert?,
-                       sslkey:  @cfg.ssl_key?)
+      def connect
+        options = {
+          adapter:  @cfg.adapter,
+          host:     @cfg.host?,
+          user:     @cfg.user?,
+          password: @cfg.password?,
+          database: @cfg.database,
+          ssl_mode: @cfg.ssl_mode?
+        }
+        if @cfg.with_ssl?
+          options.merge!(sslca:   @cfg.ssl_ca?,
+                         sslcert: @cfg.ssl_cert?,
+                         sslkey:  @cfg.ssl_key?)
+        end
+        Sequel.connect(options)
+      rescue Sequel::AdapterNotFound => e
+        raise OxidizedError, "SQL adapter gem not installed: " + e.message
       end
-      Sequel.connect(options)
-    rescue Sequel::AdapterNotFound => e
-      raise OxidizedError, "SQL adapter gem not installed: " + e.message
     end
   end
 end

data/lib/oxidized/version.rb

@@ -1,6 +1,6 @@
 module Oxidized
-  VERSION = '0.30.1'.freeze
-  VERSION_FULL = '0.30.1'.freeze
+  VERSION = '0.32.0'.freeze
+  VERSION_FULL = '0.32.0'.freeze
   def self.version_set
     version_full = %x(git describe --tags).chop rescue ""
     version      = %x(git describe --tags --abbrev=0).chop rescue ""

data/oxidized.gemspec

@@ -13,35 +13,42 @@ Gem::Specification.new do |s|
   s.summary           = 'feeble attempt at rancid'
   s.description       = 'software to fetch configuration from network devices and store them'
   s.rubyforge_project = s.name
-  s.files             = %x(git ls-files -z).split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.files             = %x(git ls-files -z).split("\x0").reject { |f| f.match(/^(test|spec|features)\//) }
   s.executables       = %w[oxidized]
   s.require_path      = 'lib'
 
   s.metadata['rubygems_mfa_required'] = 'true'
 
-  s.required_ruby_version = '>= 3.0'
+  s.required_ruby_version = '>= 3.1'
 
-  s.add_runtime_dependency 'asetus',       '~> 0.1'
-  s.add_runtime_dependency 'bcrypt_pbkdf', '~> 1.0'
-  s.add_runtime_dependency 'ed25519',      '~> 1.2'
-  s.add_runtime_dependency 'net-ftp',      '~> 0.2'
-  s.add_runtime_dependency 'net-scp',      '~> 4.0'
-  s.add_runtime_dependency 'net-ssh',      '~> 7.1'
-  s.add_runtime_dependency 'net-telnet',   '~> 0.2'
-  s.add_runtime_dependency 'psych',        '~> 3.3.2'
-  s.add_runtime_dependency 'rugged',       '~> 1.6'
-  s.add_runtime_dependency 'slop',         '~> 4.6'
+  s.add_dependency 'asetus',               '~> 0.1'
+  s.add_dependency 'bcrypt_pbkdf',         '~> 1.0'
+  s.add_dependency 'ed25519',              '~> 1.2'
+  s.add_dependency 'net-ftp',              '~> 0.2'
+  s.add_dependency 'net-http-digest_auth', '~> 1.4'
+  s.add_dependency 'net-scp',              '~> 4.1'
+  s.add_dependency 'net-ssh',              '~> 7.3'
+  s.add_dependency 'net-telnet',           '~> 0.2'
+  s.add_dependency 'ostruct',              '~> 0.6'
+  s.add_dependency 'psych',                '~> 5.0'
+  s.add_dependency 'rugged',               '~> 1.6'
+  s.add_dependency 'slop',                 '~> 4.6'
 
   s.add_development_dependency 'bundler',             '~> 2.2'
-  s.add_development_dependency 'git',                 '~> 1'
-  s.add_development_dependency 'minitest',            '~> 5.18'
+  s.add_development_dependency 'git',                 '~> 2'
+  s.add_development_dependency 'minitest',            '~> 5.25.4'
   s.add_development_dependency 'mocha',               '~> 2.1'
-  s.add_development_dependency 'pry',                 '~> 0.14.2'
+  s.add_development_dependency 'pry',                 '~> 0.15.0'
   s.add_development_dependency 'rake',                '~> 13.0'
-  s.add_development_dependency 'rubocop',             '~> 1.62.0'
-  s.add_development_dependency 'rubocop-minitest',    '~> 0.35.0'
+  s.add_development_dependency 'rubocop',             '~> 1.72.0'
+  s.add_development_dependency 'rubocop-minitest',    '~> 0.36.0'
   s.add_development_dependency 'rubocop-rake',        '~> 0.6.0'
+  s.add_development_dependency 'rubocop-sequel',      '~> 0.3.3'
   s.add_development_dependency 'simplecov',           '~> 0.22.0'
   s.add_development_dependency 'simplecov-cobertura', '~> 2.1.0'
-  s.add_development_dependency 'simplecov-html',      '~> 0.12.3'
+  s.add_development_dependency 'simplecov-html',      '~> 0.13.1'
+
+  # Dependencies on optional libraries, used for unit tests & development
+  s.add_development_dependency 'oxidized-web',        '>= 0.15.0'
+  s.add_development_dependency 'sequel',              '~> 5.88.0'
 end