topological_inventory-providers-common 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 022a317ceef535823e38291efad54d0a498905a6149e9fe75e5fc3ee036b7277
+  data.tar.gz: ade50e0e3fc05c34ec151673c076e9b521bd7d3426fd82480dc2c30e00406d7c
+SHA512:
+  metadata.gz: a9e7f2af5e8affba2ae20513a0cc802aef9bdbdcefbac6ab3aff174fc4ee7cbf4fcafb3138845bcf90a9a11b8e4e363b605b549bf3cfefe90c847be842b04a63
+  data.tar.gz: 2c41976bc692984a032847e5b2e24fa7c30ec4c0f9da14032fc31e9dbf69da0ebad0c33d88ea08f7dee700c9a84f5e174b682073ae7a6f9e9cfb021230bbe88f

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/bundler.d/
+/Gemfile.lock
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,10 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.7
+  - 2.6.5
+before_install:
+  - 'echo ''gem: --no-ri --no-rdoc --no-document'' > ~/.gemrc'
+  - gem install bundler

data/CHANGELOG.md

@@ -0,0 +1,19 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.0.1] - 2020-05-06
+### Changed
+
+Add logging method to base collector #18
+manageiq-loggers to 0.5.0 #19
+manageiq-loggers to >= 0.4.2 #20
+
+## [1.0.0] - 2020-03-19
+### Initial release to rubygems.org
+
+[Unreleased]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.1...HEAD
+[1.0.1]: https://github.com/RedHatInsights/topological_inventory-providers-common/compare/v1.0.0...v1.0.1
+[1.0.0]: https://github.com/RedHatInsights/topological_inventory-providers-common/releases/v1.0.0

data/Gemfile

@@ -0,0 +1,15 @@
+source "https://rubygems.org"
+
+plugin 'bundler-inject', '~> 1.1'
+require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundler-inject") rescue nil
+
+# Specify your gem's dependencies in topological_inventory-providers-common.gemspec
+gemspec
+
+gem "sources-api-client", "~> 1.0"
+gem "topological_inventory-ingress_api-client", "~> 1.0"
+
+group :development, :test do
+  gem 'rake', '~> 12.0.0'
+  gem 'pry-byebug'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Martin Slemr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# TopologicalInventory::Providers::Common
+
+Common code for topological-inventory collectors and Operation workers.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "topological_inventory/providers/common"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/topological_inventory/providers/common.rb

@@ -0,0 +1,15 @@
+require "topological_inventory/providers/common/version"
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/operations/processor"
+require "topological_inventory/providers/common/operations/endpoint_client"
+require "topological_inventory/providers/common/collectors_pool"
+require "topological_inventory/providers/common/collector"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Error < StandardError; end
+
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector.rb

@@ -0,0 +1,178 @@
+require "active_support/inflector"
+require "concurrent"
+require "topological_inventory-ingress_api-client"
+require "topological_inventory/providers/common/collector/inventory_collection_storage"
+require "topological_inventory/providers/common/collector/inventory_collection_wrapper"
+require "topological_inventory/providers/common/collector/parser"
+require "topological_inventory/providers/common/save_inventory/saver"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        # @param poll_time [Integer] Waiting between collecting loops. Irrelevant for standalone_mode: true
+        # @param standalone_mode [Boolean] T/F if collector is created by collectors_pool
+        def initialize(source, default_limit: 1_000, poll_time: 30, standalone_mode: true)
+          self.collector_threads = Concurrent::Map.new
+          self.finished          = Concurrent::AtomicBoolean.new(false)
+          self.poll_time         = poll_time
+          self.limits            = Hash.new(default_limit)
+          self.queue             = Queue.new
+          self.source            = source
+          self.standalone_mode   = standalone_mode
+        end
+
+        def collect!
+          start_collector_threads
+
+          until finished? do
+            ensure_collector_threads
+
+            notices = []
+            notices << queue.pop until queue.empty?
+
+            targeted_refresh(notices) unless notices.empty?
+
+            standalone_mode ? sleep(poll_time) : stop
+          end
+        end
+
+        def stop
+          finished.value = true
+        end
+
+        protected
+
+        attr_accessor :collector_threads, :finished, :limits,
+                      :poll_time, :queue, :source, :standalone_mode
+
+        def finished?
+          finished.value
+        end
+
+        def entity_types
+          endpoint_types.flat_map { |endpoint| send("#{endpoint}_entity_types") }
+        end
+
+        # Should be overriden by subclass
+        # Entity types collected from endpoints
+        def endpoint_types
+          %w()
+        end
+
+        def start_collector_threads
+          entity_types.each do |entity_type|
+            next if collector_threads[entity_type]&.alive?
+
+            collector_threads[entity_type] = start_collector_thread(entity_type)
+          end
+        end
+
+        def ensure_collector_threads
+          start_collector_threads
+        end
+
+        def start_collector_thread(entity_type)
+          connection = connection_for_entity_type(entity_type)
+          return if connection.nil?
+
+          Thread.new do
+            collector_thread(connection, entity_type)
+          end
+        end
+
+        # Connection to endpoint for each entity type
+        def connection_for_entity_type(_entity_type)
+          raise NotImplementedError
+        end
+
+        # Thread's main for collecting one entity type's data
+        def collector_thread(_connection, _entity_type)
+          raise NotImplementedError
+        end
+
+        # @optional
+        # Listen to notices from threads
+        def targeted_refresh(notices)
+        end
+
+        # @param refresh_state_part_collected_at [Time] when this payload is collected (for [Core]:RefreshStatePart)
+        # @param refresh_state_part_sent_at [Time] when this payload is sent (for [Core]:RefreshStatePart)
+        def save_inventory(collections,
+                           inventory_name,
+                           schema,
+                           refresh_state_uuid = nil,
+                           refresh_state_part_uuid = nil,
+                           refresh_state_part_collected_at = nil,
+                           refresh_state_part_sent_at = Time.now.utc)
+          return 0 if collections.empty?
+
+          SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
+            :inventory => TopologicalInventoryIngressApiClient::Inventory.new(
+              :name                            => inventory_name,
+              :schema                          => TopologicalInventoryIngressApiClient::Schema.new(:name => schema),
+              :source                          => source,
+              :collections                     => collections,
+              :refresh_state_uuid              => refresh_state_uuid,
+              :refresh_state_part_uuid         => refresh_state_part_uuid,
+              :refresh_state_part_collected_at => refresh_state_part_collected_at,
+              :refresh_state_part_sent_at      => refresh_state_part_sent_at
+            )
+          )
+        rescue => e
+          response_body    = e.response_body if e.respond_to? :response_body
+          response_headers = e.response_headers if e.respond_to? :response_headers
+          logger.error("Error when sending payload to Ingress API. Error message: #{e.message}. Body: #{response_body}. Header: #{response_headers}")
+          raise e
+        end
+
+        # @param refresh_state_started_at [Time] when collecting of this entity type is started (for [Core]:RefreshState)
+        # @param refresh_state_sent_at [Time] when this payload is sent (for [Core]:RefreshState)
+        def sweep_inventory(inventory_name,
+                            schema,
+                            refresh_state_uuid,
+                            total_parts,
+                            sweep_scope,
+                            refresh_state_started_at = nil,
+                            refresh_state_sent_at = Time.now.utc)
+          return if !total_parts || sweep_scope.empty?
+
+          SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
+            :inventory => TopologicalInventoryIngressApiClient::Inventory.new(
+              :name                     => inventory_name,
+              :schema                   => TopologicalInventoryIngressApiClient::Schema.new(:name => schema),
+              :source                   => source,
+              :collections              => [],
+              :refresh_state_uuid       => refresh_state_uuid,
+              :total_parts              => total_parts,
+              :sweep_scope              => sweep_scope,
+              :refresh_state_started_at => refresh_state_started_at,
+              :refresh_state_sent_at    => refresh_state_sent_at
+            )
+          )
+        rescue => e
+          response_body    = e.response_body if e.respond_to? :response_body
+          response_headers = e.response_headers if e.respond_to? :response_headers
+          logger.error("Error when sending payload to Ingress API. Error message: #{e.message}. Body: #{response_body}. Header: #{response_headers}")
+          raise e
+        end
+
+        def inventory_name
+          "Default"
+        end
+
+        def schema_name
+          "Default"
+        end
+
+        def ingress_api_client
+          TopologicalInventoryIngressApiClient::DefaultApi.new
+        end
+
+        def log_external_url(url)
+          logger.info("[EXTERNAL URL] #{url}")
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb

@@ -0,0 +1,69 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class InventoryCollectionStorage
+          attr_accessor :data
+
+          delegate :values, :to => :data
+
+          def initialize
+            @data = {}
+          end
+
+          def add_collection(name, overwrite: true)
+            return @data[name] if !@data[name].nil? && !overwrite
+
+            if ingress_api_model_exists?(name)
+              @data[name] ||= InventoryCollectionWrapper.new(:name => name)
+            else
+              raise NameError, "TopologicalInventoryIngressApiClient::#{name.to_s.classify} doesn't exist"
+            end
+          end
+
+          # Creates collection automatically
+          def [](name)
+            add_collection(name, :overwrite => false)
+          end
+
+          # @return [Array<Symbol>] array of InventoryCollection object names of the persister
+          def inventory_collections_names
+            @data.keys
+          end
+
+          def method_missing(method_name, *arguments, &block)
+            add_collection(method_name, :overwrite => false) # init collection if not exist
+
+            if inventory_collections_names.include?(method_name)
+              self.class.define_collections_reader(method_name)
+              send(method_name)
+            else
+              super
+            end
+          end
+
+          # @return [Boolean] true if InventoryCollection with passed method_name name is defined
+          def respond_to_missing?(method_name, _include_private = false)
+            ingress_api_model_exists?(method_name) || super
+          end
+
+          protected
+
+          def ingress_api_model_exists?(method_name)
+            class_name = "TopologicalInventoryIngressApiClient::#{method_name.to_s.classify}"
+
+            # nil test is not enough due to sometimes weird namespace autoloading
+            class_name.safe_constantize.to_s == class_name
+          end
+
+          # Defines a new attr reader returning InventoryCollection object
+          def self.define_collections_reader(collection_key)
+            define_method(collection_key) do
+              add_collection(collection_key, :overwrite => false)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb

@@ -0,0 +1,25 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class InventoryCollectionWrapper < TopologicalInventoryIngressApiClient::InventoryCollection
+          def initialize(name:)
+            super(:name => name, :data => [])
+          end
+
+          def build(properties)
+            obj = get_model.new(properties)
+            data << obj
+            obj
+          end
+
+          protected
+
+          def get_model
+            "TopologicalInventoryIngressApiClient::#{name.to_s.classify}".constantize
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/parser.rb

@@ -0,0 +1,34 @@
+require "active_support/inflector"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class Parser
+          attr_accessor :collections, :resource_timestamp
+
+          delegate :add_collection, :to => :collections
+
+          def initialize
+            @collections = InventoryCollectionStorage.new
+
+            self.resource_timestamp = Time.now.utc
+          end
+
+          def lazy_find(collection, reference, ref: :manager_ref)
+            return if reference.kind_of?(String) && reference.blank?
+
+            # Don't make lazy link if all reference values are blank
+            return if reference.kind_of?(Hash) && reference.values.select { |val| val.to_s.present? }.blank?
+
+            TopologicalInventoryIngressApiClient::InventoryObjectLazy.new(
+              :inventory_collection_name => collection,
+              :reference                 => reference,
+              :ref                       => ref,
+            )
+          end
+        end
+      end
+    end
+  end
+end