inventory_refresh 0.0.1

data/lib/inventory_refresh/inventory_collection/reference.rb

@@ -0,0 +1,96 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    class Reference
+      attr_reader :full_reference, :keys, :ref, :stringified_reference
+
+      delegate :[], :to => :full_reference
+
+      # @param data [Hash, String] Data needed for creating the reference
+      # @param ref [String] Name of the reference (and of the index associated)
+      # @param keys [Array<Symbol>] Attribute/column names of the reference, that are used as indexes of the passed
+      #        data hash
+      def initialize(data, ref, keys)
+        @full_reference = build_full_reference(data, keys)
+        @ref            = ref
+        @keys           = keys
+
+        @nested_secondary_index = keys.select { |key| full_reference[key].kind_of?(InventoryRefresh::InventoryObjectLazy) }.any? do |key|
+          !full_reference[key].primary?
+        end
+
+        @stringified_reference = self.class.build_stringified_reference(full_reference, keys)
+      end
+
+      # Return true if reference is to primary index, false otherwise. Reference is primary, only if all the nested
+      # references are primary.
+      #
+      # @return [Boolean] true if reference is to primary index, false otherwise
+      def primary?
+        ref == :manager_ref && !nested_secondary_index
+      end
+
+      def nested_secondary_index?
+        nested_secondary_index
+      end
+
+      class << self
+        # Builds string uuid from passed Hash and keys
+        #
+        # @param hash [Hash] Hash data
+        # @param keys [Array<Symbol>] Indexes into the Hash data
+        # @return [String] Concatenated values on keys from data
+        def build_stringified_reference(hash, keys)
+          stringify_reference(keys.map { |attribute| hash[attribute].to_s })
+        end
+
+        # Builds string uuid from passed Object and keys
+        #
+        # @param record [ApplicationRecord] ActiveRecord record
+        # @param keys [Array<Symbol>] Indexes into the Hash data
+        # @return [String] Concatenated values on keys from data
+        def build_stringified_reference_for_record(record, keys)
+          stringify_reference(keys.map { |attribute| record.public_send(attribute).to_s })
+        end
+
+        # Returns passed array joined by stringify_joiner
+        #
+        # @param reference [Array<String>]
+        # @return [String] Passed array joined by stringify_joiner
+        def stringify_reference(reference)
+          reference.join(stringify_joiner)
+        end
+
+        private
+
+        # Returns joiner for string UIID
+        #
+        # @return [String] Joiner for string UIID
+        def stringify_joiner
+          "__"
+        end
+      end
+
+      private
+
+      # @return Returns true if reference has nested references that are not pointing to primary index, but to
+      #         secondary indexes.
+      attr_reader :nested_secondary_index
+
+      # Returns original Hash, or build hash out of passed string
+      #
+      # @param data [Hash, String] Passed data
+      # @param keys [Array<Symbol>] Keys for the reference
+      # @return [Hash] Original Hash, or build hash out of passed string
+      def build_full_reference(data, keys)
+        if data.kind_of?(Hash)
+          data
+        else
+          raise "Please provide Hash as a reference, :manager_ref count includes more than 1 attribute. keys: #{keys}, data: #{data}" if keys.size > 1
+          {keys.first => data}
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/references_storage.rb

@@ -0,0 +1,106 @@
+module InventoryRefresh
+  class InventoryCollection
+    class ReferencesStorage
+      # @return [Hash] A set of InventoryObjects manager_uuids, which tells us which InventoryObjects were
+      #         referenced by other InventoryObjects using a lazy_find.
+      attr_reader :references
+
+      # @return [Set] A set of InventoryObject attributes names, which tells us InventoryObject attributes
+      #         were referenced by other InventoryObject objects using a lazy_find with :key.
+      attr_reader :attribute_references
+
+      def initialize(index_proxy)
+        @index_proxy                   = index_proxy
+        @references                    = {}
+        @references[primary_index_ref] = {}
+        @attribute_references          = Set.new
+      end
+
+      # Adds reference to the storage. The reference can be already existing, otherwise we attempt to build it.
+      #
+      # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference
+      #        object, or data we will build the reference object from. For InventoryCollection with :manager_ref size
+      #        bigger than 1, it's required to pass a Hash.
+      # @param key [String] If the reference comes from a InventoryObjectLazy, pointing to specific attribute using :key
+      #        we want to record what attribute was referenced.
+      # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
+      #        index.
+      def add_reference(reference_data, key: nil, ref: nil)
+        reference           = build_reference(reference_data, ref)
+        specific_references = references[reference.ref] ||= {}
+
+        specific_references[reference.stringified_reference] = reference
+
+        # If we access an attribute of the value, using a :key, we want to keep a track of that
+        attribute_references << key if key
+      end
+
+      # Adds reference to the storage. The reference can be already existing, otherwise we attempt to build it. This is
+      # simplified version of add_reference, not allowing to define :key or :ref.
+      #
+      # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference
+      #        object, or data we will build the reference object from. For InventoryCollection with :manager_ref size
+      #        bigger than 1, it's required to pass a Hash.
+      def <<(reference_data)
+        add_reference(reference_data)
+      end
+
+      # Adds array of references to the storage. The reference can be already existing, otherwise we attempt to build
+      # it.
+      #
+      # @param references_array [Array] Array of reference objects acceptable by add_reference method.
+      # @param ref [Symbol] A key to specific reference, if it's a reference pointing to something else than primary
+      #        index.
+      # @return [InventoryRefresh::InventoryCollection::ReferencesStorage] Returns self
+      def merge!(references_array, ref: nil)
+        references_array.each { |reference_data| add_reference(reference_data, :ref => ref) }
+        self
+      end
+
+      # @return [Hash{String => InventoryRefresh::InventoryCollection::Reference}] Hash of indexed Reference objects
+      def primary_references
+        references[primary_index_ref]
+      end
+
+      # Builds a Reference object
+      #
+      # @param reference_data [InventoryRefresh::InventoryCollection::References, Hash, Object] Either existing Reference
+      #        object, or data we will build the reference object from. For InventoryCollection with :manager_ref size
+      #        bigger than 1, it's required to pass a Hash.
+      def build_reference(reference_data, ref = nil)
+        ref ||= primary_index_ref
+        return reference_data if reference_data.kind_of?(::InventoryRefresh::InventoryCollection::Reference)
+
+        ::InventoryRefresh::InventoryCollection::Reference.new(reference_data, ref, named_ref(ref))
+      end
+
+      # Builds string uuid from passed Hash and keys
+      #
+      # @param hash [Hash] Hash data
+      # @param keys [Array<Symbol>] Indexes into the Hash data
+      # @return [String] Concatenated values on keys from data
+      def build_stringified_reference(hash, keys)
+        ::InventoryRefresh::InventoryCollection::Reference.build_stringified_reference(hash, keys)
+      end
+
+      # Builds string uuid from passed Object and keys
+      #
+      # @param record [ApplicationRecord] ActiveRecord record
+      # @param keys [Array<Symbol>] Indexes into the Hash data
+      # @return [String] Concatenated values on keys from data
+      def build_stringified_reference_for_record(record, keys)
+        ::InventoryRefresh::InventoryCollection::Reference.build_stringified_reference_for_record(record, keys)
+      end
+
+      private
+
+      # @return [InventoryRefresh::InventoryCollection::Index::Proxy] Index::Proxy object associated to this reference
+      #         storage
+      attr_reader :index_proxy
+
+      delegate :named_ref,
+               :primary_index_ref,
+               :to => :index_proxy
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/scanner.rb

@@ -0,0 +1,117 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    class Scanner
+      class << self
+        # Scanning inventory_collections for dependencies and references, storing the results in the inventory_collections
+        # themselves. Dependencies are needed for building a graph, references are needed for effective DB querying, where
+        # we can load all referenced objects of some InventoryCollection by one DB query.
+        #
+        # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] Array of InventoryCollection objects
+        def scan!(inventory_collections)
+          indexed_inventory_collections = inventory_collections.index_by(&:name)
+
+          inventory_collections.each do |inventory_collection|
+            new(inventory_collection, indexed_inventory_collections).scan!
+          end
+
+          inventory_collections.each do |inventory_collection|
+            inventory_collection.dependencies.each do |dependency|
+              dependency.dependees << inventory_collection
+            end
+          end
+        end
+      end
+
+      attr_reader :inventory_collection, :indexed_inventory_collections
+
+      # Boolean helpers the scanner uses from the :inventory_collection
+      delegate :inventory_object_lazy?,
+               :inventory_object?,
+               :targeted?,
+               :to => :inventory_collection
+
+      # Methods the scanner uses from the :inventory_collection
+      delegate :data,
+               :find_or_build,
+               :manager_ref,
+               :saver_strategy,
+               :to => :inventory_collection
+
+      # The data scanner modifies inside of the :inventory_collection
+      delegate :attribute_references,
+               :data_collection_finalized=,
+               :dependency_attributes,
+               :targeted_scope,
+               :parent_inventory_collections,
+               :parent_inventory_collections=,
+               :references,
+               :transitive_dependency_attributes,
+               :to => :inventory_collection
+
+      def initialize(inventory_collection, indexed_inventory_collections)
+        @inventory_collection          = inventory_collection
+        @indexed_inventory_collections = indexed_inventory_collections
+      end
+
+      def scan!
+        # Scan InventoryCollection InventoryObjects and store the results inside of the InventoryCollection
+        data.each do |inventory_object|
+          scan_inventory_object!(inventory_object)
+
+          if targeted? && parent_inventory_collections.blank?
+            # We want to track what manager_uuids we should query from a db, for the targeted refresh
+            targeted_scope << inventory_object.reference
+          end
+        end
+
+        # Transform :parent_inventory_collections symbols to InventoryCollection objects
+        if parent_inventory_collections.present?
+          self.parent_inventory_collections = parent_inventory_collections.map do |inventory_collection_index|
+            ic = indexed_inventory_collections[inventory_collection_index]
+            if ic.nil?
+              raise "Can't find InventoryCollection #{inventory_collection_index} from #{inventory_collection}" if targeted?
+            else
+              # Add parent_inventory_collection as a dependency, so e.g. disconnect is done in a right order
+              (dependency_attributes[:__parent_inventory_collections] ||= Set.new) << ic
+              ic
+            end
+          end.compact
+        end
+
+        # Mark InventoryCollection as finalized aka. scanned
+        self.data_collection_finalized = true
+      end
+
+      private
+
+      def scan_inventory_object!(inventory_object)
+        inventory_object.data.each do |key, value|
+          if value.kind_of?(Array)
+            value.each { |val| scan_inventory_object_attribute!(key, val) }
+          else
+            scan_inventory_object_attribute!(key, value)
+          end
+        end
+      end
+
+      def scan_inventory_object_attribute!(key, value)
+        return if !inventory_object_lazy?(value) && !inventory_object?(value)
+        value_inventory_collection = value.inventory_collection
+
+        # Storing attributes and their dependencies
+        (dependency_attributes[key] ||= Set.new) << value_inventory_collection if value.dependency?
+
+        # Storing a reference in the target inventory_collection, then each IC knows about all the references and can
+        # e.g. load all the referenced uuids from a DB
+        value_inventory_collection.add_reference(value.reference, :key => value.key)
+
+        if inventory_object_lazy?(value)
+          # Storing if attribute is a transitive dependency, so a lazy_find :key results in dependency
+          transitive_dependency_attributes << key if value.transitive_dependency?
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/serialization.rb

@@ -0,0 +1,140 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    class Serialization
+      delegate :all_manager_uuids,
+               :build,
+               :targeted_scope,
+               :data,
+               :inventory_object_lazy?,
+               :inventory_object?,
+               :name,
+               :skeletal_primary_index,
+               :to => :inventory_collection
+
+      attr_reader :inventory_collection
+
+      # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want the storage
+      #        for
+      def initialize(inventory_collection)
+        @inventory_collection = inventory_collection
+      end
+
+      # Loads InventoryCollection data from it's serialized form into existing InventoryCollection object
+      #
+      # @param inventory_objects_data [Hash] Serialized InventoryCollection as Hash
+      # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      def from_hash(inventory_objects_data, available_inventory_collections)
+        targeted_scope.merge!(inventory_objects_data["manager_uuids"].map(&:symbolize_keys!))
+
+        inventory_objects_data['data'].each do |inventory_object_data|
+          build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
+        end
+
+        inventory_objects_data['partial_data'].each do |inventory_object_data|
+          skeletal_primary_index.build(hash_to_data(inventory_object_data, available_inventory_collections).symbolize_keys!)
+        end
+
+        # TODO(lsmola) add support for all_manager_uuids serialization
+        # self.all_manager_uuids = inventory_objects_data['all_manager_uuids']
+      end
+
+      # Serializes InventoryCollection's data storage into Array of Hashes
+      #
+      # @return [Hash] Serialized InventoryCollection object into Hash
+      def to_hash
+        {
+          :name              => name,
+          # TODO(lsmola) we do not support nested references here, should we?
+          :manager_uuids     => targeted_scope.primary_references.values.map(&:full_reference),
+          :all_manager_uuids => all_manager_uuids,
+          :data              => data.map { |x| data_to_hash(x.data) },
+          :partial_data      => skeletal_primary_index.index_data.map { |x| data_to_hash(x.data) },
+        }
+      end
+
+      private
+
+      # Converts InventoryRefresh::InventoryObject or InventoryRefresh::InventoryObjectLazy into Hash
+      #
+      # @param value [InventoryRefresh::InventoryObject, InventoryRefresh::InventoryObjectLazy] InventoryObject or a lazy link
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Serialized InventoryRefresh::InventoryObjectLazy
+      def lazy_relation_to_hash(value, depth = 0)
+        {
+          :type                        => "InventoryRefresh::InventoryObjectLazy",
+          :inventory_collection_name   => value.inventory_collection.name,
+          :reference                   => data_to_hash(value.reference.full_reference, depth + 1),
+          :ref                         => value.reference.ref,
+          :key                         => value.try(:key),
+          :default                     => value.try(:default),
+          :transform_nested_lazy_finds => value.try(:transform_nested_lazy_finds)
+        }
+      end
+
+      # Converts Hash to InventoryRefresh::InventoryObjectLazy
+      #
+      # @param value [Hash] Serialized InventoryObject or a lazy link
+      # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [InventoryRefresh::InventoryObjectLazy] Lazy link created from hash
+      def hash_to_lazy_relation(value, available_inventory_collections, depth = 0)
+        inventory_collection = available_inventory_collections[value['inventory_collection_name'].try(:to_sym)]
+        raise "Couldn't build lazy_link #{value} the inventory_collection_name was not found" if inventory_collection.blank?
+
+        inventory_collection.lazy_find(
+          hash_to_data(value['reference'], available_inventory_collections, depth + 1).symbolize_keys!,
+          :ref                         => value['ref'].try(:to_sym),
+          :key                         => value['key'].try(:to_sym),
+          :default                     => value['default'],
+          :transform_nested_lazy_finds => value['transform_nested_lazy_finds']
+        )
+      end
+
+      # Converts Hash to attributes usable for building InventoryObject
+      #
+      # @param hash [Hash] Serialized InventoryObject data
+      # @param available_inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of available
+      #        InventoryCollection objects
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Hash with data usable for building InventoryObject
+      def hash_to_data(hash, available_inventory_collections, depth = 0)
+        raise "Nested lazy_relation of #{inventory_collection} is too deep, left processing: #{hash}" if depth > 20
+
+        hash.transform_values do |value|
+          if value.kind_of?(Hash) && value['type'] == "InventoryRefresh::InventoryObjectLazy"
+            hash_to_lazy_relation(value, available_inventory_collections, depth)
+          elsif value.kind_of?(Array) && value.first.kind_of?(Hash) && value.first['type'] == "InventoryRefresh::InventoryObjectLazy"
+            # TODO(lsmola) do we need to compact it sooner? What if first element is nil? On the other hand, we want to
+            # deprecate this Vm HABTM assignment because it's not effective
+            value.compact.map { |x| hash_to_lazy_relation(x, available_inventory_collections, depth) }
+          else
+            value
+          end
+        end
+      end
+
+      # Transforms data of the InventoryObject or Reference to InventoryObject into Hash
+      #
+      # @param data [Hash] Data of the InventoryObject or Reference to InventoryObject
+      # @param depth [Integer] Depth of nesting for nested lazy link
+      # @return [Hash] Serialized InventoryObject or Reference data into Hash
+      def data_to_hash(data, depth = 0)
+        raise "Nested lazy_relation of #{inventory_collection} is too deep, left processing: #{data}" if depth > 20
+
+        data.transform_values do |value|
+          if inventory_object_lazy?(value) || inventory_object?(value)
+            lazy_relation_to_hash(value, depth)
+          elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
+            value.compact.map { |x| lazy_relation_to_hash(x, depth) }
+          else
+            value
+          end
+        end
+      end
+    end
+  end
+end