inventory_refresh 0.0.1

data/lib/inventory_refresh/inventory_collection/data_storage.rb

@@ -0,0 +1,178 @@
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    class DataStorage
+      # @return [Array<InventoryObject>] objects of the InventoryCollection in an Array
+      attr_accessor :data
+
+      attr_reader :index_proxy, :inventory_collection
+
+      delegate :each, :size, :to => :data
+
+      delegate :primary_index,
+               :build_primary_index_for,
+               :build_secondary_indexes_for,
+               :named_ref,
+               :skeletal_primary_index,
+               :to => :index_proxy
+
+      delegate :association_to_foreign_key_mapping,
+               :default_values,
+               :inventory_object?,
+               :inventory_object_lazy?,
+               :manager_ref,
+               :new_inventory_object,
+               :to => :inventory_collection
+
+      # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object we want the storage
+      #        for
+      # @param secondary_refs [Hash] Secondary_refs in format {:name_of_the_ref => [:attribute1, :attribute2]}
+      def initialize(inventory_collection, secondary_refs)
+        @inventory_collection = inventory_collection
+        @data                 = []
+
+        @index_proxy = InventoryRefresh::InventoryCollection::Index::Proxy.new(inventory_collection, secondary_refs)
+      end
+
+      # Adds passed InventoryObject into the InventoryCollection's storage
+      #
+      # @param inventory_object [InventoryRefresh::InventoryObject]
+      # @return [InventoryRefresh::InventoryCollection] Returns current InventoryCollection, to allow chaining
+      def <<(inventory_object)
+        if inventory_object.manager_uuid.present? && !primary_index.find(inventory_object.manager_uuid)
+          data << inventory_object
+
+          # TODO(lsmola) Maybe we do not need the secondary indexes here?
+          # Maybe we should index it like LocalDb indexes, on demand, and storing what was
+          # indexed? Maybe we should allow only lazy access and no direct find from a parser. Since for streaming
+          # refresh, things won't be parsed together and no full state will be taken.
+          build_primary_index_for(inventory_object)
+          build_secondary_indexes_for(inventory_object)
+        end
+        inventory_collection
+      end
+
+      alias push <<
+
+      # Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
+      #
+      # @param manager_uuid [String] manager_uuid of the InventoryObject
+      # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject
+      def find_or_build(manager_uuid)
+        raise "The uuid consists of #{manager_ref.size} attributes, please find_or_build_by method" if manager_ref.size > 1
+
+        find_or_build_by(manager_ref.first => manager_uuid)
+      end
+
+      # (see #build)
+      def find_or_build_by(hash)
+        build(hash)
+      end
+
+      # Finds InventoryObject.
+      #
+      # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the InventoryCollection
+      # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
+      def find_in_data(hash)
+        _hash, _uuid, inventory_object = primary_index_scan(hash)
+        inventory_object
+      end
+
+      # Finds of builds a new InventoryObject. By building it, we also put in into the InventoryCollection's storage.
+      #
+      # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
+      #        InventoryCollection
+      # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
+      def build(hash)
+        hash, uuid, inventory_object = primary_index_scan(hash)
+
+        # Return InventoryObject if found in primary index
+        return inventory_object unless inventory_object.nil?
+
+        # We will take existing skeletal record, so we don't duplicate references for saving. We can have duplicated
+        # reference from local_db index, (if we are using .find in parser, that causes N+1 db queries), but that is ok,
+        # since that one is not being saved.
+        inventory_object = skeletal_primary_index.delete(uuid)
+
+        # We want to update the skeletal record with actual data
+        inventory_object&.assign_attributes(hash)
+
+        # Build the InventoryObject
+        inventory_object ||= new_inventory_object(enrich_data(hash))
+
+        # Store new InventoryObject and return it
+        push(inventory_object)
+        inventory_object
+      end
+
+      # Finds of builds a new InventoryObject with incomplete data.
+      #
+      # @param hash [Hash] Hash that needs to contain attributes defined in :manager_ref of the
+      #        InventoryCollection
+      # @return [InventoryRefresh::InventoryObject] Found or built InventoryObject object
+      def build_partial(hash)
+        skeletal_primary_index.build(hash)
+      end
+
+      # Returns array of built InventoryObject objects
+      #
+      # @return [Array<InventoryRefresh::InventoryObject>] Array of built InventoryObject objects
+      def to_a
+        data
+      end
+
+      def to_hash
+        InventoryRefresh::InventoryCollection::Serialization.new(inventory_collection).to_hash
+      end
+
+      def from_hash(inventory_objects_data, available_inventory_collections)
+        InventoryRefresh::InventoryCollection::Serialization
+          .new(inventory_collection)
+          .from_hash(inventory_objects_data, available_inventory_collections)
+      end
+
+      private
+
+      # Scans primary index for existing InventoryObject, that would be equivalent to passed hash. It also returns
+      # enriched data and uuid, so we do not have to compute it multiple times.
+      #
+      # @param hash [Hash] Attributes for the InventoryObject
+      # @return [Array(Hash, String, InventoryRefresh::InventoryObject)] Returns enriched data, uuid and InventoryObject
+      # if found (otherwise nil)
+      def primary_index_scan(hash)
+        hash = enrich_data(hash)
+
+        assert_all_keys_present(hash)
+        assert_only_primary_index(hash)
+
+        uuid = ::InventoryRefresh::InventoryCollection::Reference.build_stringified_reference(hash, named_ref)
+        return hash, uuid, primary_index.find(uuid)
+      end
+
+      def assert_all_keys_present(hash)
+        if manager_ref.any? { |x| !hash.key?(x) }
+          raise "Needed find_or_build_by keys are: #{manager_ref}, data provided: #{hash}"
+        end
+      end
+
+      def assert_only_primary_index(data)
+        named_ref.each do |key|
+          if data[key].kind_of?(InventoryRefresh::InventoryObjectLazy) && !data[key].primary?
+            raise "Wrong index for key :#{key}, all references under this index must point to default :ref called"\
+                  " :manager_ref. Any other :ref is not valid. This applies also to nested lazy links."
+          end
+        end
+      end
+
+      # Returns new hash enriched by (see InventoryRefresh::InventoryCollection#default_values) hash
+      #
+      # @param hash [Hash] Input hash
+      # @return [Hash] Enriched hash by (see InventoryRefresh::InventoryCollection#default_values)
+      def enrich_data(hash)
+        # This is 25% faster than default_values.merge(hash)
+        {}.merge!(default_values).merge!(hash)
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/graph.rb

@@ -0,0 +1,170 @@
+module InventoryRefresh
+  class InventoryCollection
+    class Graph < ::InventoryRefresh::Graph
+      # @param nodes [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
+      def initialize(nodes)
+        super(nodes)
+
+        assert_inventory_collections(nodes)
+      end
+
+      # Turns the graph into DAG (Directed Acyclic Graph)
+      #
+      # @return [InventoryRefresh::InventoryCollection::Graph] Graph object modified into DAG
+      def build_directed_acyclic_graph!
+        ################################################################################################################
+        ## Description of an algorithm for building DAG
+        ################################################################################################################
+        # Terms:
+        ##############
+        # Fixed Edges - Are edges that cannot be removed from Graph, in our case these are edges caused by attributes
+        #               that has to be present before saving the record. These are attributes that are part of the
+        #               record identity (unique index of the DB record) and attributes validated for presence.
+        # Feedback Edge Set - Is a set of edges that are causing a cycle in the graph
+        # DAG - Directed Acyclic Graph
+        #
+        # Alghoritm:
+        ##############
+        # Building a Feedback Edge Set:
+        # We will be building DAG from a Graph containing cycles, the algorithm is building a Feedback Edge Set by
+        # adding edges to a DAG made from Fixed Edges, while checking if by adding a new edge we haven't created
+        # a cycle in the graph.
+        # Converting original graph to DAG:
+        # Using the Feedback Edge Set, we remove the attributes causing cycles from the original graph. This way, we
+        # get a DAG, but the DAG is missing attributes.
+        # Using the Feedback Edge Set we create new Nodes, containing only removed attributes in a step before. These
+        # nodes will be attached to Graph according to their dependencies. By saving these nodes, we will add the
+        # missing relations.
+        ################################################################################################################
+
+        # Assert that Fixed edges do not have a cycle, we cannot move with fixed edges, so exception is thrown here
+        assert_graph!(fixed_edges)
+
+        # Collect Feedback Edge (Arc) Set
+        feedback_edge_set = build_feedback_edge_set(edges, fixed_edges)
+
+        # We will build a DAG using the Feedback Edge (Arc) Set. All edges from this set has to be removed, and the
+        # edges are transferred to newly created nodes.
+        convert_to_dag!(nodes, feedback_edge_set)
+
+        # And assert again we've really built a DAG
+        # TODO(lsmola) And if the result causes a cycle, we should repeat the build_dag method, with a max
+        # depth 10. We should throw a warning maybe asking for simplifying the interconnections in the models.
+        assert_graph!(edges)
+
+        self.nodes = sort_nodes(nodes)
+
+        self
+      end
+
+      private
+
+      # Sort the nodes putting child nodes as last. Child nodes are InventoryCollection objects having
+      # :parent_inventory_collections definition.
+      #
+      # @param nodes [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
+      # @return [Array<InventoryRefresh::InventoryCollection>] Sorted list of Inventory collection nodes
+      def sort_nodes(nodes)
+        # Separate to root nodes and child nodes, where child nodes are determined by having parent_inventory_collections
+        root_nodes, child_nodes = nodes.partition { |node| node.parent_inventory_collections.blank? }
+        # And order it that root nodes comes first, that way the disconnect of the child nodes should be always done as
+        # a part of the root nodes, as intended.
+        root_nodes + child_nodes
+      end
+
+      # Asserts all nodes are of class ::InventoryRefresh::InventoryCollection
+      #
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
+      def assert_inventory_collections(inventory_collections)
+        inventory_collections.each do |inventory_collection|
+          unless inventory_collection.kind_of?(::InventoryRefresh::InventoryCollection)
+            raise "A InventoryRefresh::SaveInventory needs a InventoryCollection object, it got: #{inventory_collection.inspect}"
+          end
+        end
+      end
+
+      # Converts the graph into DAG
+      #
+      # @param nodes [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
+      # @param feedback_edge_set [Array<Array>] Is a set of edges that are causing a cycle in the graph
+      # @return [InventoryRefresh::InventoryCollection::Graph] Graph object modified into DAG
+      def convert_to_dag!(nodes, feedback_edge_set)
+        new_nodes = []
+        inventory_collection_transformations = {}
+        nodes.each do |inventory_collection|
+          feedback_dependencies = feedback_edge_set.select { |e| e.second == inventory_collection }.map(&:first)
+          attrs                 = inventory_collection.dependency_attributes_for(feedback_dependencies)
+
+          next if attrs.blank?
+
+          new_inventory_collection = inventory_collection.clone
+
+          # Add inventory_collection as a dependency of the new_inventory_collection, so we make sure it runs after
+          # TODO(lsmola) add a nice dependency_attributes setter? It's used also in actualize_dependencies method
+          new_inventory_collection.dependency_attributes[:__feedback_edge_set_parent] = Set.new([inventory_collection])
+          new_nodes << new_inventory_collection
+
+          inventory_collection.blacklist_attributes!(attrs)
+          new_inventory_collection.whitelist_attributes!(attrs)
+
+          # Store a simple hash for transforming inventory_collection to new_inventory_collection
+          inventory_collection_transformations[inventory_collection] = new_inventory_collection
+        end
+
+        all_nodes = nodes + new_nodes
+
+        # If we remove an attribute that was a dependency of another node, we need to move also the
+        # dependency. So e.g. floating_ip depends on network_port's attribute vm, but we move that attribute to new
+        # network_port inventory_collection. We will need to move also the dependency to point to the new
+        # inventory_collection.
+        #
+        # So we have to go through all dependencies that loads a key, which is the moved attribute. We can get a list
+        # of attributes that are using a key from transitive_dependency_attributes, from there we can get a list of
+        # dependencies. And from the list of dependencies, we can check which ones were moved just by looking into
+        # inventory_collection_transformations.
+        all_nodes.each do |inventory_collection|
+          inventory_collection.transitive_dependency_attributes.each do |transitive_dependency_attribute|
+            transitive_dependencies = inventory_collection.dependency_attributes[transitive_dependency_attribute]
+            next if transitive_dependencies.blank?
+
+            transitive_dependencies.map! do |dependency|
+              transformed_dependency = inventory_collection_transformations[dependency]
+              transformed_dependency.blank? ? dependency : transformed_dependency
+            end
+          end
+        end
+
+        # Add the new InventoryCollections to the list of nodes our our graph
+        construct_graph!(all_nodes)
+      end
+
+      # Build edges by introspecting the passed array of InventoryCollection objects
+      #
+      # @param inventory_collections [Array<InventoryRefresh::InventoryCollection>] List of Inventory collection nodes
+      # @return [Array<Array>] Nested array representing edges
+      def build_edges(inventory_collections)
+        edges            = []
+        transitive_edges = []
+        fixed_edges = []
+        inventory_collections.each do |inventory_collection|
+          inventory_collection.dependencies.each do |dependency|
+            fixed_edges << [dependency, inventory_collection] if inventory_collection.fixed_dependencies.include?(dependency)
+            if inventory_collection.dependency_attributes_for([dependency]).any? { |x| inventory_collection.transitive_dependency_attributes.include?(x) }
+              # The condition checks if the dependency is a transitive dependency, in other words a InventoryObjectLazy with :key
+              # pointing to another object.
+              transitive_edges << [dependency, inventory_collection]
+            else
+              edges << [dependency, inventory_collection]
+            end
+          end
+        end
+        # We put transitive edges to the end. Transitive edge is e.g.: given graph (X,Y,Z), we have a lazy link, from X
+        # to Y, making edge (Y, X), using a :key pointing to Z. Which means that also edge from Y to Z (Z, Y) exists.
+        # If the edge (Z, Y) is placed before (Y, X), we process it first. Then the edge (Y, X), causing hidden
+        # transitive relation X to Z (it's hidden because edge (Z, X) is not present), is processed as last and we do a
+        # more effective cycle removal if needed.
+        return edges + transitive_edges, fixed_edges
+      end
+    end
+  end
+end

data/lib/inventory_refresh/inventory_collection/index/proxy.rb

@@ -0,0 +1,230 @@
+require "inventory_refresh/inventory_collection/index/type/data"
+require "inventory_refresh/inventory_collection/index/type/local_db"
+require "inventory_refresh/inventory_collection/index/type/skeletal"
+require "active_support/core_ext/module/delegation"
+
+module InventoryRefresh
+  class InventoryCollection
+    module Index
+      class Proxy
+        attr_reader :skeletal_primary_index
+
+        # @param inventory_collection [InventoryRefresh::InventoryCollection] InventoryCollection object owning the proxy
+        # @param secondary_refs [Hash] Secondary_refs in format {:name_of_the_ref => [:attribute1, :attribute2]}
+        def initialize(inventory_collection, secondary_refs = {})
+          @inventory_collection = inventory_collection
+
+          @primary_ref    = {primary_index_ref => @inventory_collection.manager_ref}
+          @secondary_refs = secondary_refs
+          @all_refs       = @primary_ref.merge(@secondary_refs)
+
+          @data_indexes     = {}
+          @local_db_indexes = {}
+
+          @all_refs.each do |index_name, attribute_names|
+            @data_indexes[index_name] = InventoryRefresh::InventoryCollection::Index::Type::Data.new(
+              inventory_collection,
+              index_name,
+              attribute_names
+            )
+
+            @local_db_indexes[index_name] = InventoryRefresh::InventoryCollection::Index::Type::LocalDb.new(
+              inventory_collection,
+              index_name,
+              attribute_names,
+              @data_indexes[index_name]
+            )
+          end
+
+          @skeletal_primary_index = InventoryRefresh::InventoryCollection::Index::Type::Skeletal.new(
+            inventory_collection,
+            :skeletal_primary_index_ref,
+            named_ref,
+            primary_index
+          )
+        end
+
+        # Builds primary index for passed InventoryObject
+        #
+        # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject we want to index
+        # @return [InventoryRefresh::InventoryObject] Passed InventoryObject
+        def build_primary_index_for(inventory_object)
+          # Building the object, we need to provide all keys of a primary index
+
+          assert_index(inventory_object.data, primary_index_ref)
+          primary_index.store_index_for(inventory_object)
+        end
+
+        def build_secondary_indexes_for(inventory_object)
+          secondary_refs.keys.each do |ref|
+            data_index(ref).store_index_for(inventory_object)
+          end
+        end
+
+        def reindex_secondary_indexes!
+          data_indexes.each do |ref, index|
+            next if ref == primary_index_ref
+
+            index.reindex!
+          end
+        end
+
+        def primary_index
+          data_index(primary_index_ref)
+        end
+
+        def find(reference, ref: primary_index_ref)
+          # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
+          # TODO(lsmola) this method should return lazy too, the rest of the finders should be deprecated
+          return if reference.nil?
+          assert_index(reference, ref)
+
+          reference = inventory_collection.build_reference(reference, ref)
+
+          case strategy
+          when :local_db_find_references, :local_db_cache_all
+            local_db_index_find(reference)
+          when :local_db_find_missing_references
+            find_in_data_or_skeletal_index(reference) || local_db_index_find(reference)
+          else
+            find_in_data_or_skeletal_index(reference)
+          end
+        end
+
+        def find_by(manager_uuid_hash, ref: primary_index_ref)
+          # TODO(lsmola) deprecate this, it's enough to have find method
+          find(manager_uuid_hash, :ref => ref)
+        end
+
+        def lazy_find_by(manager_uuid_hash, ref: primary_index_ref, key: nil, default: nil)
+          # TODO(lsmola) deprecate this, it's enough to have lazy_find method
+
+          lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
+        end
+
+        def lazy_find(manager_uuid, ref: primary_index_ref, key: nil, default: nil, transform_nested_lazy_finds: false)
+          # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
+          # access the data
+          # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
+          return if manager_uuid.nil?
+          assert_index(manager_uuid, ref)
+
+          ::InventoryRefresh::InventoryObjectLazy.new(inventory_collection,
+                                                    manager_uuid,
+                                                    :ref                         => ref,
+                                                    :key                         => key,
+                                                    :default                     => default,
+                                                    :transform_nested_lazy_finds => transform_nested_lazy_finds)
+        end
+
+        def named_ref(ref = primary_index_ref)
+          all_refs[ref]
+        end
+
+        def primary_index_ref
+          :manager_ref
+        end
+
+        private
+
+        delegate :association_to_foreign_key_mapping,
+                 :build_stringified_reference,
+                 :parallel_safe?,
+                 :strategy,
+                 :to => :inventory_collection
+
+        attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
+
+        def find_in_data_or_skeletal_index(reference)
+          if parallel_safe?
+            # With parallel safe strategies, we create skeletal nodes that we can look for
+            data_index_find(reference) || skeletal_index_find(reference)
+          else
+            data_index_find(reference)
+          end
+        end
+
+        def skeletal_index_find(reference)
+          # Find in skeletal index, but we are able to create skeletal index only for primary indexes
+          skeletal_primary_index.find(reference.stringified_reference) if reference.primary?
+        end
+
+        def data_index_find(reference)
+          data_index(reference.ref).find(reference.stringified_reference)
+        end
+
+        def local_db_index_find(reference)
+          local_db_index(reference.ref).find(reference)
+        end
+
+        def data_index(name)
+          data_indexes[name] || raise("Index :#{name} not defined for #{inventory_collection}")
+        end
+
+        def local_db_index(name)
+          local_db_indexes[name] || raise("Index :#{name} not defined for #{inventory_collection}")
+        end
+
+        def missing_keys(data_keys, ref)
+          named_ref(ref) - data_keys
+        end
+
+        def required_index_keys_present?(data_keys, ref)
+          missing_keys(data_keys, ref).empty?
+        end
+
+        def assert_relation_keys(data, ref)
+          named_ref(ref).each do |key|
+            # Skip if the key is not a foreign key
+            next unless association_to_foreign_key_mapping[key]
+            # Skip if data on key are nil or InventoryObject or InventoryObjectLazy
+            next if data[key].nil? || data[key].kind_of?(InventoryRefresh::InventoryObject) || data[key].kind_of?(InventoryRefresh::InventoryObjectLazy)
+            # Raise error since relation must be nil or InventoryObject or InventoryObjectLazy
+            raise "Wrong index for key :#{key}, the value must be of type Nil or InventoryObject or InventoryObjectLazy, got: #{data[key]}"
+          end
+        end
+
+        def assert_index_exists(ref)
+          raise "Index :#{ref} doesn't exist on #{inventory_collection}" if named_ref(ref).nil?
+        end
+
+        def assert_index(manager_uuid, ref)
+          # TODO(lsmola) do we need some production logging too? Maybe the refresh log level could drive this
+          # Let' do this really slick development and test env, but disable for production, since the checks are pretty
+          # slow.
+          # TODO: return if Rails.env.production?
+
+          if manager_uuid.kind_of?(InventoryRefresh::InventoryCollection::Reference)
+            # InventoryRefresh::InventoryCollection::Reference has been already asserted, skip
+          elsif manager_uuid.kind_of?(Hash)
+            # Test te index exists
+            assert_index_exists(ref)
+
+            # Test we are sending all keys required for the index
+            unless required_index_keys_present?(manager_uuid.keys, ref)
+              raise "Finder has missing keys for index :#{ref}, missing indexes are: #{missing_keys(manager_uuid.keys, ref)}"
+            end
+            # Test that keys, that are relations, are nil or InventoryObject or InventoryObjectlazy class
+            assert_relation_keys(manager_uuid, ref)
+          else
+            # Test te index exists
+            assert_index_exists(ref)
+
+            # Check that other value (possibly String or Integer)) has no composite index
+            if named_ref(ref).size > 1
+              right_format = "collection.find(#{named_ref(ref).map { |x| ":#{x} => 'X'" }.join(", ")}"
+
+              raise "The index :#{ref} has composite index, finder has to be called as: #{right_format})"
+            end
+
+            # Assert the that possible relation is nil or InventoryObject or InventoryObjectlazy class
+            assert_relation_keys({named_ref(ref).first => manager_uuid}, ref)
+          end
+        rescue => e
+          #_log.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of: #{inventory_collection}")
+          raise e
+        end
+      end
+    end
+  end
+end