topological_inventory-providers-common 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0f2d31b001af718d1e0cb85a61754fcc3a7e74073718083aaf2706190906606e
+  data.tar.gz: 28c1849fa09d45ec3f33600d43db393521dbd70c1c51775f5092cba1bceba91d
+SHA512:
+  metadata.gz: a0831719dc7226ba7aabb254b6d43fe154a779e69bac479c4d4fc6be3bf9c1990d86ffb9c4ee0a44fdd3695edd2ee83a522109bcf04cec2694c272ab5393999a
+  data.tar.gz: fc9e8ccc5478379cde68f8b6fd6dcdd00db2d0193c692dbf1563b6d4e0d753b58eca9fe9eed30688f8bf0a6194b3369b44f4ecfce5fcd008df632f654b78d1b6

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/bundler.d/
+/Gemfile.lock
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,10 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.4.5
+  - 2.5.3
+before_install:
+  - 'echo ''gem: --no-ri --no-rdoc --no-document'' > ~/.gemrc'
+  - gem install bundler

data/Gemfile

@@ -0,0 +1,15 @@
+source "https://rubygems.org"
+
+plugin 'bundler-inject', '~> 1.1'
+require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundler-inject") rescue nil
+
+# Specify your gem's dependencies in topological_inventory-providers-common.gemspec
+gemspec
+
+gem "sources-api-client", "~> 1.0"
+gem "topological_inventory-ingress_api-client", "~> 1.0"
+
+group :development, :test do
+  gem 'rake', '~> 12.0.0'
+  gem 'pry-byebug'
+end

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Martin Slemr
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,3 @@
+# TopologicalInventory::Providers::Common
+
+Common code for topological-inventory collectors and Operation workers.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "topological_inventory/providers/common"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/topological_inventory/providers/common.rb

@@ -0,0 +1,15 @@
+require "topological_inventory/providers/common/version"
+require "topological_inventory/providers/common/logging"
+require "topological_inventory/providers/common/operations/processor"
+require "topological_inventory/providers/common/operations/endpoint_client"
+require "topological_inventory/providers/common/collectors_pool"
+require "topological_inventory/providers/common/collector"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Error < StandardError; end
+
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector.rb

@@ -0,0 +1,162 @@
+require "active_support/inflector"
+require "concurrent"
+require "topological_inventory-ingress_api-client"
+require "topological_inventory/providers/common/collector/inventory_collection_storage"
+require "topological_inventory/providers/common/collector/inventory_collection_wrapper"
+require "topological_inventory/providers/common/collector/parser"
+require "topological_inventory/providers/common/save_inventory/saver"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        # @param poll_time [Integer] Waiting between collecting loops. Irrelevant for standalone_mode: true
+        # @param standalone_mode [Boolean] T/F if collector is created by collectors_pool
+        def initialize(source, default_limit: 1_000, poll_time: 30, standalone_mode: true)
+          self.collector_threads = Concurrent::Map.new
+          self.finished          = Concurrent::AtomicBoolean.new(false)
+          self.poll_time         = poll_time
+          self.limits            = Hash.new(default_limit)
+          self.queue             = Queue.new
+          self.source            = source
+          self.standalone_mode   = standalone_mode
+        end
+
+        def collect!
+          start_collector_threads
+
+          until finished? do
+            ensure_collector_threads
+
+            notices = []
+            notices << queue.pop until queue.empty?
+
+            targeted_refresh(notices) unless notices.empty?
+
+            standalone_mode ? sleep(poll_time) : stop
+          end
+        end
+
+        def stop
+          finished.value = true
+        end
+
+        protected
+
+        attr_accessor :collector_threads, :finished, :limits,
+                      :poll_time, :queue, :source, :standalone_mode
+
+        def finished?
+          finished.value
+        end
+
+        def entity_types
+          endpoint_types.flat_map { |endpoint| send("#{endpoint}_entity_types") }
+        end
+
+        # Should be overriden by subclass
+        # Entity types collected from endpoints
+        def endpoint_types
+          %w()
+        end
+
+        def start_collector_threads
+          entity_types.each do |entity_type|
+            next if collector_threads[entity_type]&.alive?
+
+            collector_threads[entity_type] = start_collector_thread(entity_type)
+          end
+        end
+
+        def ensure_collector_threads
+          start_collector_threads
+        end
+
+        def start_collector_thread(entity_type)
+          connection = connection_for_entity_type(entity_type)
+          return if connection.nil?
+
+          Thread.new do
+            collector_thread(connection, entity_type)
+          end
+        end
+
+        # Connection to endpoint for each entity type
+        def connection_for_entity_type(_entity_type)
+          raise NotImplementedError
+        end
+
+        # Thread's main for collecting one entity type's data
+        def collector_thread(_connection, _entity_type)
+          raise NotImplementedError
+        end
+
+        # @optional
+        # Listen to notices from threads
+        def targeted_refresh(notices)
+        end
+
+        def save_inventory(collections,
+                           inventory_name,
+                           schema,
+                           refresh_state_uuid = nil,
+                           refresh_state_part_uuid = nil)
+          return 0 if collections.empty?
+
+          SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
+            :inventory => TopologicalInventoryIngressApiClient::Inventory.new(
+              :name                    => inventory_name,
+              :schema                  => TopologicalInventoryIngressApiClient::Schema.new(:name => schema),
+              :source                  => source,
+              :collections             => collections,
+              :refresh_state_uuid      => refresh_state_uuid,
+              :refresh_state_part_uuid => refresh_state_part_uuid,
+            )
+          )
+        rescue => e
+          response_body    = e.response_body if e.respond_to? :response_body
+          response_headers = e.response_headers if e.respond_to? :response_headers
+          logger.error("Error when sending payload to Ingress API. Error message: #{e.message}. Body: #{response_body}. Header: #{response_headers}")
+          raise e
+        end
+
+        def sweep_inventory(inventory_name,
+                            schema,
+                            refresh_state_uuid,
+                            total_parts,
+                            sweep_scope)
+          return if !total_parts || sweep_scope.empty?
+
+          SaveInventory::Saver.new(:client => ingress_api_client, :logger => logger).save(
+            :inventory => TopologicalInventoryIngressApiClient::Inventory.new(
+              :name               => inventory_name,
+              :schema             => TopologicalInventoryIngressApiClient::Schema.new(:name => schema),
+              :source             => source,
+              :collections        => [],
+              :refresh_state_uuid => refresh_state_uuid,
+              :total_parts        => total_parts,
+              :sweep_scope        => sweep_scope,
+            )
+          )
+        rescue => e
+          response_body    = e.response_body if e.respond_to? :response_body
+          response_headers = e.response_headers if e.respond_to? :response_headers
+          logger.error("Error when sending payload to Ingress API. Error message: #{e.message}. Body: #{response_body}. Header: #{response_headers}")
+          raise e
+        end
+
+        def inventory_name
+          "Default"
+        end
+
+        def schema_name
+          "Default"
+        end
+
+        def ingress_api_client
+          TopologicalInventoryIngressApiClient::DefaultApi.new
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb

@@ -0,0 +1,69 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class InventoryCollectionStorage
+          attr_accessor :data
+
+          delegate :values, :to => :data
+
+          def initialize
+            @data = {}
+          end
+
+          def add_collection(name, overwrite: true)
+            return @data[name] if !@data[name].nil? && !overwrite
+
+            if ingress_api_model_exists?(name)
+              @data[name] ||= InventoryCollectionWrapper.new(:name => name)
+            else
+              raise NameError, "TopologicalInventoryIngressApiClient::#{name.to_s.classify} doesn't exist"
+            end
+          end
+
+          # Creates collection automatically
+          def [](name)
+            add_collection(name, :overwrite => false)
+          end
+
+          # @return [Array<Symbol>] array of InventoryCollection object names of the persister
+          def inventory_collections_names
+            @data.keys
+          end
+
+          def method_missing(method_name, *arguments, &block)
+            add_collection(method_name, :overwrite => false) # init collection if not exist
+
+            if inventory_collections_names.include?(method_name)
+              self.class.define_collections_reader(method_name)
+              send(method_name)
+            else
+              super
+            end
+          end
+
+          # @return [Boolean] true if InventoryCollection with passed method_name name is defined
+          def respond_to_missing?(method_name, _include_private = false)
+            ingress_api_model_exists?(method_name) || super
+          end
+
+          protected
+
+          def ingress_api_model_exists?(method_name)
+            class_name = "TopologicalInventoryIngressApiClient::#{method_name.to_s.classify}"
+
+            # nil test is not enough due to sometimes weird namespace autoloading
+            class_name.safe_constantize.to_s == class_name
+          end
+
+          # Defines a new attr reader returning InventoryCollection object
+          def self.define_collections_reader(collection_key)
+            define_method(collection_key) do
+              add_collection(collection_key, :overwrite => false)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb

@@ -0,0 +1,25 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class InventoryCollectionWrapper < TopologicalInventoryIngressApiClient::InventoryCollection
+          def initialize(name:)
+            super(:name => name, :data => [])
+          end
+
+          def build(properties)
+            obj = get_model.new(properties)
+            data << obj
+            obj
+          end
+
+          protected
+
+          def get_model
+            "TopologicalInventoryIngressApiClient::#{name.to_s.classify}".constantize
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collector/parser.rb

@@ -0,0 +1,34 @@
+require "active_support/inflector"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class Collector
+        class Parser
+          attr_accessor :collections, :resource_timestamp
+
+          delegate :add_collection, :to => :collections
+
+          def initialize
+            @collections = InventoryCollectionStorage.new
+
+            self.resource_timestamp = Time.now.utc
+          end
+
+          def lazy_find(collection, reference, ref: :manager_ref)
+            return if reference.kind_of?(String) && reference.blank?
+
+            # Don't make lazy link if all reference values are blank
+            return if reference.kind_of?(Hash) && reference.values.select { |val| val.to_s.present? }.blank?
+
+            TopologicalInventoryIngressApiClient::InventoryObjectLazy.new(
+              :inventory_collection_name => collection,
+              :reference                 => reference,
+              :ref                       => ref,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/collectors_pool.rb

@@ -0,0 +1,135 @@
+require "config"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class CollectorsPool
+        SECRET_FILENAME = "credentials".freeze
+
+        def initialize(config_name, metrics, collector_poll_time: 60, thread_pool_size: 2)
+          self.config_name         = config_name
+          self.collector_status    = Concurrent::Map.new
+          self.metrics             = metrics
+          self.collector_poll_time = collector_poll_time
+          self.secrets             = nil
+          self.thread_pool         = Concurrent::FixedThreadPool.new(thread_pool_size)
+          self.updated_at          = {}
+        end
+
+        def run!
+          loop do
+            reload_config
+            reload_secrets
+
+            # Secret is deployed just after config map, we should wait for it
+            queue_collectors if secrets_newer_than_config?
+
+            sleep(5)
+          end
+        end
+
+        def stop!
+          collectors.each_value(&:stop)
+
+          thread_pool.shutdown
+          # Wait for end of collectors to ensure metrics are stopped after them
+          thread_pool.wait_for_termination
+        end
+
+        protected
+
+        attr_accessor :collectors, :collector_poll_time, :collector_status, :thread_pool, :config_name,
+                      :metrics, :secrets, :updated_at
+
+        def reload_config
+          config_file = File.join(path_to_config, "#{sanitize_filename(config_name)}.yml")
+          raise "Configuration file #{config_file} doesn't exist" unless File.exist?(config_file)
+
+          ::Config.load_and_set_settings(config_file)
+        end
+
+        def reload_secrets
+          path = File.join(path_to_secrets, SECRET_FILENAME)
+          raise "Secrets file missing at #{path}" unless File.exists?(path)
+          file         = File.read(path)
+          self.secrets = JSON.parse(file)
+        end
+
+        # @param [Hash] source from Settings
+        # @return [Hash|nil] {"username":, "password":}
+        def secrets_for_source(source)
+          secrets[source.source]
+        end
+
+        def queue_collectors
+          ::Settings.sources.to_a.each do |source|
+            # Skip if collector is running/queued or just finished
+            next if queued_or_updated_recently?(source)
+
+            # Check if secrets for this source are present
+            next if (source_secret = secrets_for_source(source)).nil?
+
+            # Check if necessary endpoint/auth data are not blank (provider specific)
+            next unless source_valid?(source, source_secret)
+
+            collector_status[source.source] = {:status => :queued}
+            # Add source to collector's queue
+            thread_pool.post do
+              begin
+                collector = new_collector(source, source_secret)
+                collector.collect!
+              ensure
+                collector_status[source.source] = {:status => :ready, :last_updated_at => Time.now}
+              end
+            end
+          end
+        end
+
+        def queued_or_updated_recently?(source)
+          return false if collector_status[source.source].nil?
+          return true if collector_status[source.source][:status] == :queued
+
+          if (last_updated_at = collector_status[source.source][:last_updated_at]).nil?
+            # should never happen
+            last_updated_at = Time.now
+            collector_status[source.source] = {:status => :ready, :last_updated_at => last_updated_at}
+          end
+
+          last_updated_at > Time.now - collector_poll_time.to_i
+        end
+
+        def secrets_newer_than_config?
+          return false if ::Settings.updated_at.nil? || secrets["updated_at"].nil?
+
+          updated_at[:config] = Time.parse(::Settings.updated_at)
+          updated_at[:secret] = Time.parse(secrets["updated_at"])
+
+          logger.info("Reloading Sources data => Config [updated_at: #{updated_at[:config].to_s}], Secrets [updated at: #{updated_at[:secret]}]") if updated_at[:config] <= updated_at[:secret]
+
+          updated_at[:config] <= updated_at[:secret]
+        end
+
+        def source_valid?(source, secret)
+          true
+        end
+
+        def path_to_config
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def path_to_secrets
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+
+        def sanitize_filename(filename)
+          # Remove any character that aren't 0-9, A-Z, or a-z, / or -
+          filename.gsub(/[^0-9A-Z\/\-]/i, '_')
+        end
+
+        def new_collector(source, source_secret)
+          raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/logging.rb

@@ -0,0 +1,22 @@
+require "manageiq/loggers"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      class << self
+        attr_writer :logger
+      end
+
+      def self.logger
+        @logger ||= ManageIQ::Loggers::Container.new
+        @logger
+      end
+
+      module Logging
+        def logger
+          TopologicalInventory::Providers::Common.logger
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/endpoint_client.rb

@@ -0,0 +1,62 @@
+require "topological_inventory/providers/common/operations/topology_api_client"
+require "topological_inventory/providers/common/operations/sources_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class EndpointClient
+          include TopologyApiClient
+
+          def initialize(source_id, task_id, identity = nil)
+            self.identity   = identity
+            self.source_id  = source_id
+            self.task_id    = task_id
+          end
+
+          def order_service(service_offering, service_plan, order_params)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def source_ref_of(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def wait_for_provision_complete(source_id, endpoint_svc_instance, context = {})
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          def provisioned_successfully?(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          # Endpoint for conversion of provisioned service's status to
+          # TopologicalInventory Task's status
+          def task_status_for(endpoint_svc_instance)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass"
+          end
+
+          private
+
+          attr_accessor :identity, :task_id, :source_id
+
+          def sources_api
+            @sources_api ||= SourcesApiClient.new(identity)
+          end
+
+          def default_endpoint
+            @default_endpoint ||= sources_api.fetch_default_endpoint(source_id)
+          end
+
+          def authentication
+            @authentication ||= sources_api.fetch_authentication(source_id, default_endpoint)
+          end
+
+          def verify_ssl_mode
+            default_endpoint.verify_ssl ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/processor.rb

@@ -0,0 +1,135 @@
+require "topological_inventory/providers/common/operations/topology_api_client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class Processor
+          include Logging
+          include TopologyApiClient
+
+          SLEEP_POLL = 10
+          POLL_TIMEOUT = 1800
+
+          def self.process!(message)
+            model, method = message.headers['message_type'].to_s.split(".")
+            new(model, method, message.payload).process
+          end
+
+          # @param payload [Hash] https://github.com/ManageIQ/topological_inventory-api/blob/master/app/controllers/api/v0/service_plans_controller.rb#L32-L41
+          def initialize(model, method, payload, metrics = nil)
+            self.model           = model
+            self.method          = method
+            self.params          = payload["params"]
+            self.identity        = payload["request_context"]
+            self.metrics         = metrics
+          end
+
+          def process
+            logger.info("Processing #{model}##{method} [#{params}]...")
+            result = order_service(params)
+            logger.info("Processing #{model}##{method} [#{params}]...Complete")
+
+            result
+          end
+
+          private
+
+          attr_accessor :identity, :model, :method, :metrics, :params
+
+          def endpoint_client(source_id, task_id, identity)
+            raise NotImplementedError, "#{__method__} must be implemented in a subclass as kind of TopologicalInventory::Providers::Common::EndpointClient class"
+          end
+
+          def order_service(params)
+            task_id, service_offering_id, service_plan_id, order_params = params.values_at("task_id", "service_offering_id", "service_plan_id", "order_params")
+
+            service_plan     = topology_api_client.show_service_plan(service_plan_id) if service_plan_id.present?
+            service_offering_id = service_plan.service_offering_id if service_offering_id.nil? && service_plan.present?
+            service_offering = topology_api_client.show_service_offering(service_offering_id)
+
+            source_id        = service_offering.source_id
+            client = endpoint_client(source_id, task_id, identity)
+
+            logger.info("Ordering #{service_offering.name}...")
+            remote_service_instance = client.order_service(service_offering, service_plan.presence, order_params)
+            logger.info("Ordering #{service_offering.name}...Complete")
+
+            poll_order_complete_thread(task_id, source_id, remote_service_instance)
+          rescue StandardError => err
+            metrics&.record_error
+            logger.error("[Task #{task_id}] Ordering error: #{err}\n#{err.backtrace.join("\n")}")
+            update_task(task_id, :state => "completed", :status => "error", :context => {:error => err.to_s})
+          end
+
+          def poll_order_complete_thread(task_id, source_id, remote_svc_instance)
+            Thread.new do
+              begin
+                poll_order_complete(task_id, source_id, remote_svc_instance)
+              rescue StandardError => err
+                logger.error("[Task #{task_id}] Waiting for complete: #{err}\n#{err.backtrace.join("\n")}")
+                update_task(task_id, :state => "completed", :status => "warn", :context => {:error => err.to_s})
+              end
+            end
+          end
+
+          def poll_order_complete(task_id, source_id, remote_svc_instance)
+            client = endpoint_client(source_id, task_id, identity)
+
+            context = {
+              :service_instance => {
+                :source_id  => source_id,
+                :source_ref => client.source_ref_of(remote_svc_instance)
+              }
+            }
+
+            remote_svc_instance = client.wait_for_provision_complete(task_id, remote_svc_instance, context)
+
+            if client.provisioned_successfully?(remote_svc_instance)
+              if (service_instance = load_topological_svc_instance(source_id, client.source_ref_of(remote_svc_instance))).present?
+                context[:service_instance][:id] = service_instance.id
+                context[:service_instance][:url] = svc_instance_url(service_instance)
+              else
+                logger.warn("Failed to get service_instance API URL (endpoint's service instance: #{remote_svc_instance.inspect})")
+              end
+            end
+            update_task(task_id, :state => "completed", :status => client.task_status_for(remote_svc_instance), :context => context)
+          end
+
+          def load_topological_svc_instance(source_id, source_ref)
+            api = topology_api_client.api_client
+
+            count = 0
+            timeout_count = POLL_TIMEOUT / SLEEP_POLL
+
+            header_params = { 'Accept' => api.select_header_accept(['application/json']) }
+            query_params = { :'source_id' => source_id, :'source_ref' => source_ref }
+            return_type = 'ServiceInstancesCollection'
+
+            service_instance = nil
+            loop do
+              data, _status_code, _headers = api.call_api(:GET, "/service_instances",
+                                                          :header_params => header_params,
+                                                          :query_params  => query_params,
+                                                          :auth_names    => ['UserSecurity'],
+                                                          :return_type   => return_type)
+
+              service_instance = data.data&.first if data.meta.count > 0
+              break if service_instance.present?
+
+              break if (count += 1) >= timeout_count
+
+              sleep(SLEEP_POLL) # seconds
+            end
+
+            if service_instance.nil?
+              logger.error("Failed to find service_instance by source_id [#{source_id}] source_ref [#{source_ref}]")
+            end
+
+            service_instance
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/sources_api_client.rb

@@ -0,0 +1,85 @@
+require "sources-api-client"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        class SourcesApiClient < ::SourcesApiClient::ApiClient
+          INTERNAL_API_PATH = '//internal/v1.0'.freeze
+
+          def initialize(identity = nil)
+            super(::SourcesApiClient::Configuration.default)
+            self.identity = identity
+            self.api = init_default_api
+          end
+
+          def init_default_api
+            default_headers.merge!(identity) if identity.present?
+            ::SourcesApiClient::DefaultApi.new(self)
+          end
+
+          def fetch_default_endpoint(source_id)
+            endpoints = api.list_source_endpoints(source_id)&.data || []
+            endpoint = endpoints.find(&:default)
+
+            raise "Sources API: Endpoint not found! (source id: #{source_id})" if endpoint.nil?
+
+            endpoint
+          end
+
+          def fetch_authentication(source_id, default_endpoint = nil)
+            endpoint = default_endpoint || fetch_default_endpoint(source_id)
+            return if endpoint.nil?
+
+            endpoint_authentications = api.list_endpoint_authentications(endpoint.id.to_s).data || []
+            return if endpoint_authentications.empty?
+
+            auth_id = endpoint_authentications.first.id
+            fetch_authentication_with_password(auth_id)
+          end
+
+          private
+
+          attr_accessor :identity, :api, :custom_base_path
+
+          def fetch_authentication_with_password(auth_id)
+            on_internal_api do
+              local_var_path = "/authentications/#{auth_id}"
+
+              query_params = "expose_encrypted_attribute[]=password"
+
+              header_params = { 'Accept' => select_header_accept(['application/json']) }
+              return_type = 'Authentication'
+              data, _, _ = call_api(:GET, local_var_path,
+                                    :header_params => header_params,
+                                    :query_params  => query_params,
+                                    :auth_names    => ['UserSecurity'],
+                                    :return_type   => return_type)
+              data
+            end
+          end
+
+          def build_request_url(path)
+            # Add leading and trailing slashes to path
+            path = "/#{path}".gsub(/\/+/, '/')
+            URI.encode((custom_base_url || @config.base_url) + path)
+          end
+
+          def custom_base_url
+            return nil if custom_base_path.nil?
+
+            url = "#{@config.scheme}://#{[@config.host, custom_base_path].join('/').gsub(/\/+/, '/')}".sub(/\/+\z/, '')
+            URI.encode(url)
+          end
+
+          def on_internal_api
+            self.custom_base_path = INTERNAL_API_PATH
+            yield
+          ensure
+            self.custom_base_path = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/operations/topology_api_client.rb

@@ -0,0 +1,28 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module Operations
+        module TopologyApiClient
+          def topology_api_client
+            @topology_api_client ||=
+              begin
+                api_client = TopologicalInventoryApiClient::ApiClient.new
+                api_client.default_headers.merge!(identity) if identity.present?
+                TopologicalInventoryApiClient::DefaultApi.new(api_client)
+              end
+          end
+
+          def update_task(task_id, state:, status:, context:)
+            task = TopologicalInventoryApiClient::Task.new("state" => state, "status" => status, "context" => context)
+            topology_api_client.update_task(task_id, task)
+          end
+
+          def svc_instance_url(service_instance)
+            rest_api_path = '/service_instances/{id}'.sub('{' + 'id' + '}', service_instance&.id.to_s)
+            topology_api_client.api_client.build_request(:GET, rest_api_path).url
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/save_inventory/exception.rb

@@ -0,0 +1,14 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module SaveInventory
+        class Exception
+          class Error < StandardError;
+          end
+          class EntityTooLarge < Error;
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/save_inventory/saver.rb

@@ -0,0 +1,124 @@
+require "topological_inventory/providers/common/save_inventory/exception"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module SaveInventory
+        class Saver
+          # As defined in:
+          # https://github.com/zendesk/ruby-kafka/blob/02f7e2816e1130c5202764c275e36837f57ca4af/lib/kafka/protocol/message.rb#L11-L17
+          # There is at least 112 bytes that are added as a message header, so we need to keep room for that. Lets make
+          # it 200 bytes, just for sure.
+          KAFKA_RESERVED_HEADER_SIZE = 200
+
+          def initialize(client:, logger:, max_bytes: 1_000_000)
+            @client    = client
+            @logger    = logger
+            @max_bytes = max_bytes - KAFKA_RESERVED_HEADER_SIZE
+          end
+
+          attr_reader :client, :logger, :max_bytes
+
+          # @return [Integer] A total number of parts that the payload was divided into
+          def save(data)
+            inventory = data[:inventory].to_hash
+
+            inventory_json = JSON.generate(inventory)
+            if inventory_json.size < max_bytes
+              save_inventory(inventory_json)
+              return 1
+            else
+              # GC can clean this up
+              inventory_json = nil
+              return save_payload_in_batches(inventory)
+            end
+          end
+
+          private
+
+          def save_payload_in_batches(inventory)
+            parts         = 0
+            new_inventory = build_new_inventory(inventory)
+
+            inventory[:collections].each do |collection|
+              new_collection = build_new_collection(collection)
+
+              data = collection[:data].map { |x| JSON.generate(x) }
+              # Lets compute sizes of the each data item, plus 1 byte for comma
+              data_sizes = data.map { |x| x.size + 1 }
+
+              # Size of the current inventory and new_collection wrapper, plus 2 bytes for array signs
+              wrapper_size = JSON.generate(new_inventory).size + JSON.generate(new_collection).size + 2
+              total_size   = wrapper_size
+              counter      = 0
+              data_sizes.each do |data_size|
+                counter    += 1
+                total_size += data_size
+
+                if total_size > max_bytes
+                  # Remove the last data part, that caused going over the max limit
+                  counter -= 1
+
+                  # Add the entities to new collection, so the total size is below max
+                  if counter > 0
+                    new_collection[:data] = collection[:data].shift(counter)
+                    new_inventory[:collections] << new_collection
+                  end
+
+                  # Save the current batch
+                  serialize_and_save_inventory(new_inventory)
+                  parts += 1
+
+                  # Create new data containers for a new batch
+                  new_inventory  = build_new_inventory(inventory)
+                  new_collection = build_new_collection(collection)
+                  wrapper_size   = JSON.generate(new_inventory).size + JSON.generate(new_collection).size + 2
+
+                  # Start with the data part we've removed from the currently saved payload
+                  total_size = wrapper_size + data_size
+                  counter    = 1
+                end
+              end
+
+              # Store the rest of the collection
+              new_collection[:data] = collection[:data].shift(counter)
+              new_inventory[:collections] << new_collection
+            end
+
+            # save the rest
+            serialize_and_save_inventory(new_inventory)
+            parts += 1
+
+            return parts
+          end
+
+          def save_inventory(inventory_json)
+            client.save_inventory_with_http_info(inventory_json)
+          end
+
+          def serialize_and_save_inventory(inventory)
+            payload = JSON.generate(inventory)
+            if payload.size > max_bytes
+              raise Exception::EntityTooLarge,
+                    "Entity is bigger than total limit and can't be split: #{payload}"
+            end
+
+            # Save the current batch
+            save_inventory(payload)
+          end
+
+          def build_new_collection(collection)
+            {:name => collection[:name], :data => []}
+          end
+
+          def build_new_inventory(inventory)
+            new_inventory                           = inventory.clone
+            new_inventory[:refresh_state_part_uuid] = SecureRandom.uuid
+            new_inventory[:collections]             = []
+            new_inventory
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/version.rb

@@ -0,0 +1,7 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      VERSION = "0.1.0"
+    end
+  end
+end

data/topological_inventory-providers-common.gemspec

@@ -0,0 +1,35 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "topological_inventory/providers/common/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "topological_inventory-providers-common"
+  spec.version       = TopologicalInventory::Providers::Common::VERSION
+  spec.authors       = ["Martin Slemr"]
+  spec.email         = ["mslemr@redhat.com"]
+
+  spec.summary       = %q{Common classes for topological-inventory collectors/operations}
+  spec.description   = %q{Common classes for topological-inventory collectors/operations}
+  spec.homepage      = "https://github.com/slemrmartin/topological_inventory-providers-common"
+  spec.license       = "Apache-2.0"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'config', '~> 1.7', '>= 1.7.2'
+  spec.add_runtime_dependency "activesupport", "~> 5.2.2"
+  spec.add_runtime_dependency "manageiq-loggers", "~> 0.4.0", ">= 0.4.2"
+  spec.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
+  spec.add_runtime_dependency "topological_inventory-api-client", "~> 2.0"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,197 @@
+--- !ruby/object:Gem::Specification
+name: topological_inventory-providers-common
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Martin Slemr
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-02-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+- !ruby/object:Gem::Dependency
+  name: manageiq-loggers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+- !ruby/object:Gem::Dependency
+  name: topological_inventory-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Common classes for topological-inventory collectors/operations
+email:
+- mslemr@redhat.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/topological_inventory/providers/common.rb
+- lib/topological_inventory/providers/common/collector.rb
+- lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb
+- lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb
+- lib/topological_inventory/providers/common/collector/parser.rb
+- lib/topological_inventory/providers/common/collectors_pool.rb
+- lib/topological_inventory/providers/common/logging.rb
+- lib/topological_inventory/providers/common/operations/endpoint_client.rb
+- lib/topological_inventory/providers/common/operations/processor.rb
+- lib/topological_inventory/providers/common/operations/sources_api_client.rb
+- lib/topological_inventory/providers/common/operations/topology_api_client.rb
+- lib/topological_inventory/providers/common/save_inventory/exception.rb
+- lib/topological_inventory/providers/common/save_inventory/saver.rb
+- lib/topological_inventory/providers/common/version.rb
+- topological_inventory-providers-common.gemspec
+homepage: https://github.com/slemrmartin/topological_inventory-providers-common
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Common classes for topological-inventory collectors/operations
+test_files: []