topological_inventory-providers-common 1.0.1

data/lib/topological_inventory/providers/common/save_inventory/exception.rb

@@ -0,0 +1,14 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      module SaveInventory
+        class Exception
+          class Error < StandardError;
+          end
+          class EntityTooLarge < Error;
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/save_inventory/saver.rb

@@ -0,0 +1,124 @@
+require "topological_inventory/providers/common/save_inventory/exception"
+
+module TopologicalInventory
+  module Providers
+    module Common
+      module SaveInventory
+        class Saver
+          # As defined in:
+          # https://github.com/zendesk/ruby-kafka/blob/02f7e2816e1130c5202764c275e36837f57ca4af/lib/kafka/protocol/message.rb#L11-L17
+          # There is at least 112 bytes that are added as a message header, so we need to keep room for that. Lets make
+          # it 200 bytes, just for sure.
+          KAFKA_RESERVED_HEADER_SIZE = 200
+
+          def initialize(client:, logger:, max_bytes: 1_000_000)
+            @client    = client
+            @logger    = logger
+            @max_bytes = max_bytes - KAFKA_RESERVED_HEADER_SIZE
+          end
+
+          attr_reader :client, :logger, :max_bytes
+
+          # @return [Integer] A total number of parts that the payload was divided into
+          def save(data)
+            inventory = data[:inventory].to_hash
+
+            inventory_json = JSON.generate(inventory)
+            if inventory_json.size < max_bytes
+              save_inventory(inventory_json)
+              return 1
+            else
+              # GC can clean this up
+              inventory_json = nil
+              return save_payload_in_batches(inventory)
+            end
+          end
+
+          private
+
+          def save_payload_in_batches(inventory)
+            parts         = 0
+            new_inventory = build_new_inventory(inventory)
+
+            inventory[:collections].each do |collection|
+              new_collection = build_new_collection(collection)
+
+              data = collection[:data].map { |x| JSON.generate(x) }
+              # Lets compute sizes of the each data item, plus 1 byte for comma
+              data_sizes = data.map { |x| x.size + 1 }
+
+              # Size of the current inventory and new_collection wrapper, plus 2 bytes for array signs
+              wrapper_size = JSON.generate(new_inventory).size + JSON.generate(new_collection).size + 2
+              total_size   = wrapper_size
+              counter      = 0
+              data_sizes.each do |data_size|
+                counter    += 1
+                total_size += data_size
+
+                if total_size > max_bytes
+                  # Remove the last data part, that caused going over the max limit
+                  counter -= 1
+
+                  # Add the entities to new collection, so the total size is below max
+                  if counter > 0
+                    new_collection[:data] = collection[:data].shift(counter)
+                    new_inventory[:collections] << new_collection
+                  end
+
+                  # Save the current batch
+                  serialize_and_save_inventory(new_inventory)
+                  parts += 1
+
+                  # Create new data containers for a new batch
+                  new_inventory  = build_new_inventory(inventory)
+                  new_collection = build_new_collection(collection)
+                  wrapper_size   = JSON.generate(new_inventory).size + JSON.generate(new_collection).size + 2
+
+                  # Start with the data part we've removed from the currently saved payload
+                  total_size = wrapper_size + data_size
+                  counter    = 1
+                end
+              end
+
+              # Store the rest of the collection
+              new_collection[:data] = collection[:data].shift(counter)
+              new_inventory[:collections] << new_collection
+            end
+
+            # save the rest
+            serialize_and_save_inventory(new_inventory)
+            parts += 1
+
+            return parts
+          end
+
+          def save_inventory(inventory_json)
+            client.save_inventory_with_http_info(inventory_json)
+          end
+
+          def serialize_and_save_inventory(inventory)
+            payload = JSON.generate(inventory)
+            if payload.size > max_bytes
+              raise Exception::EntityTooLarge,
+                    "Entity is bigger than total limit and can't be split: #{payload}"
+            end
+
+            # Save the current batch
+            save_inventory(payload)
+          end
+
+          def build_new_collection(collection)
+            {:name => collection[:name], :data => []}
+          end
+
+          def build_new_inventory(inventory)
+            new_inventory                           = inventory.clone
+            new_inventory[:refresh_state_part_uuid] = SecureRandom.uuid
+            new_inventory[:collections]             = []
+            new_inventory
+          end
+        end
+      end
+    end
+  end
+end

data/lib/topological_inventory/providers/common/version.rb

@@ -0,0 +1,7 @@
+module TopologicalInventory
+  module Providers
+    module Common
+      VERSION = "1.0.1"
+    end
+  end
+end

data/topological_inventory-providers-common.gemspec

@@ -0,0 +1,35 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "topological_inventory/providers/common/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "topological_inventory-providers-common"
+  spec.version       = TopologicalInventory::Providers::Common::VERSION
+  spec.authors       = ["Martin Slemr"]
+  spec.email         = ["mslemr@redhat.com"]
+
+  spec.summary       = %q{Common classes for topological-inventory collectors/operations}
+  spec.description   = %q{Common classes for topological-inventory collectors/operations}
+  spec.homepage      = "https://github.com/RedHatInsights/topological_inventory-providers-common"
+  spec.license       = "Apache-2.0"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'config', '~> 1.7', '>= 1.7.2'
+  spec.add_runtime_dependency "activesupport", "~> 5.2.2"
+  spec.add_runtime_dependency "manageiq-loggers", ">= 0.4.2"
+  spec.add_runtime_dependency 'json', '~> 2.1', '>= 2.1.0'
+  spec.add_runtime_dependency "topological_inventory-api-client", "~> 3.0", ">= 3.0.1"
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

metadata

@@ -0,0 +1,198 @@
+--- !ruby/object:Gem::Specification
+name: topological_inventory-providers-common
+version: !ruby/object:Gem::Version
+  version: 1.0.1
+platform: ruby
+authors:
+- Martin Slemr
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2020-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.2.2
+- !ruby/object:Gem::Dependency
+  name: manageiq-loggers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.4.2
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.0
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: topological_inventory-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Common classes for topological-inventory collectors/operations
+email:
+- mslemr@redhat.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/topological_inventory/providers/common.rb
+- lib/topological_inventory/providers/common/collector.rb
+- lib/topological_inventory/providers/common/collector/inventory_collection_storage.rb
+- lib/topological_inventory/providers/common/collector/inventory_collection_wrapper.rb
+- lib/topological_inventory/providers/common/collector/parser.rb
+- lib/topological_inventory/providers/common/collectors_pool.rb
+- lib/topological_inventory/providers/common/logging.rb
+- lib/topological_inventory/providers/common/operations/endpoint_client.rb
+- lib/topological_inventory/providers/common/operations/processor.rb
+- lib/topological_inventory/providers/common/operations/sources_api_client.rb
+- lib/topological_inventory/providers/common/operations/topology_api_client.rb
+- lib/topological_inventory/providers/common/save_inventory/exception.rb
+- lib/topological_inventory/providers/common/save_inventory/saver.rb
+- lib/topological_inventory/providers/common/version.rb
+- topological_inventory-providers-common.gemspec
+homepage: https://github.com/RedHatInsights/topological_inventory-providers-common
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.8
+signing_key: 
+specification_version: 4
+summary: Common classes for topological-inventory collectors/operations
+test_files: []