inventory_refresh 0.0.1

data/lib/inventory_refresh/save_collection/saver/default.rb

@@ -0,0 +1,57 @@
+require "inventory_refresh/save_collection/saver/base"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    class Default < InventoryRefresh::SaveCollection::Saver::Base
+      private
+
+      # Updates the passed record with hash data and stores primary key value into inventory_object.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param hash [Hash] data we want to update the record with
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def update_record!(record, hash, inventory_object)
+        record.assign_attributes(hash.except(:id))
+        if !inventory_collection.check_changed? || record.changed?
+          record.save
+          inventory_collection.store_updated_records(record)
+        end
+
+        inventory_object.id = record.id
+      end
+
+      # Creates a new record in the DB using the passed hash data
+      #
+      # @param hash [Hash] hash with data we want to persist to DB
+      # @param inventory_object [InventoryRefresh::InventoryObject] InventoryObject instance where we will store primary
+      #        key value
+      def create_record!(hash, inventory_object)
+        record = inventory_collection.model_class.create!(hash.except(:id))
+        inventory_collection.store_created_records(record)
+
+        inventory_object.id = record.id
+      end
+
+      # Asserts we do not have duplicate records in the DB. If the record is duplicate we will destroy it.
+      #
+      # @param record [ApplicationRecord] record we want to update in DB
+      # @param index [String] manager_uuid of the record
+      # @return [Boolean] false if the record is duplicate
+      def assert_unique_record(record, index)
+        # TODO(lsmola) can go away once we indexed our DB with unique indexes
+        if unique_db_indexes.include?(index) # Include on Set is O(1)
+          # We have a duplicate in the DB, destroy it. A find_each method does automatically .order(:id => :asc)
+          # so we always keep the oldest record in the case of duplicates.
+          #_log.warn("A duplicate record was detected and destroyed, inventory_collection: "\
+          #              "'#{inventory_collection}', record: '#{record}', duplicate_index: '#{index}'")
+          record.destroy
+          return false
+        else
+          unique_db_indexes << index
+        end
+        true
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/sql_helper.rb

@@ -0,0 +1,85 @@
+require "inventory_refresh/save_collection/saver/sql_helper_update"
+require "inventory_refresh/save_collection/saver/sql_helper_upsert"
+require "active_support/concern"
+
+module InventoryRefresh::SaveCollection
+  module Saver
+    module SqlHelper
+      # TODO(lsmola) all below methods should be rewritten to arel, but we need to first extend arel to be able to do
+      # this
+
+      extend ActiveSupport::Concern
+
+      included do
+        include SqlHelperUpsert
+        include SqlHelperUpdate
+      end
+
+      # Returns quoted column name
+      # @param key [Symbol] key that is column name
+      # @returns [String] quoted column name
+      def quote_column_name(key)
+        get_connection.quote_column_name(key)
+      end
+
+      # @return [ActiveRecord::ConnectionAdapters::AbstractAdapter] ActiveRecord connection
+      def get_connection
+        ActiveRecord::Base.connection
+      end
+
+      # Builds a multiselection conditions like (table1.a = a1 AND table2.b = b1) OR (table1.a = a2 AND table2.b = b2)
+      #
+      # @param hashes [Array<Hash>] data we want to use for the query
+      # @return [String] condition usable in .where of an ActiveRecord relation
+      def build_multi_selection_query(hashes)
+        inventory_collection.build_multi_selection_condition(hashes, unique_index_columns)
+      end
+
+      # Quotes a value. For update query, the value also needs to be explicitly casted, which we can do by
+      # type_cast_for_pg param set to true.
+      #
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] ActiveRecord connection
+      # @param value [Object] value we want to quote
+      # @param name [Symbol] name of the column
+      # @param type_cast_for_pg [Boolean] true if we want to also cast the quoted value
+      # @return [String] quoted and based on type_cast_for_pg param also casted value
+      def quote(connection, value, name = nil, type_cast_for_pg = nil)
+        # TODO(lsmola) needed only because UPDATE FROM VALUES needs a specific PG typecasting, remove when fixed in PG
+        if type_cast_for_pg
+          quote_and_pg_type_cast(connection, value, name)
+        else
+          connection.quote(value)
+        end
+      rescue TypeError => e
+        #_log.error("Can't quote value: #{value}, of :#{name} and #{inventory_collection}")
+        raise e
+      end
+
+      # Quotes and type casts the value.
+      #
+      # @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] ActiveRecord connection
+      # @param value [Object] value we want to quote
+      # @param name [Symbol] name of the column
+      # @return [String] quoted and casted value
+      def quote_and_pg_type_cast(connection, value, name)
+        pg_type_cast(
+          connection.quote(value),
+          pg_types[name]
+        )
+      end
+
+      # Returns a type casted value in format needed by PostgreSQL
+      #
+      # @param value [Object] value we want to quote
+      # @param sql_type [String] PostgreSQL column type
+      # @return [String] type casted value in format needed by PostgreSQL
+      def pg_type_cast(value, sql_type)
+        if sql_type.nil?
+          value
+        else
+          "#{value}::#{sql_type}"
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/sql_helper_update.rb

@@ -0,0 +1,120 @@
+module InventoryRefresh::SaveCollection
+  module Saver
+    module SqlHelperUpdate
+      # Builds update clause for one column identified by the passed key
+      #
+      # @param key [Symbol] key that is column name
+      # @return [String] SQL clause for updating one column
+      def build_update_set_cols(key)
+        "#{quote_column_name(key)} = updated_values.#{quote_column_name(key)}"
+      end
+
+      # Build batch update query
+      #
+      # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
+      # @param hashes [Array<Hash>] data used for building a batch update sql query
+      def build_update_query(all_attribute_keys, hashes)
+        #_log.debug("Building update query for #{inventory_collection} of size #{inventory_collection.size}...")
+        # Cache the connection for the batch
+        connection = get_connection
+
+        # We want to ignore create timestamps when updating
+        all_attribute_keys_array = all_attribute_keys.to_a.delete_if { |x| %i(created_at created_on).include?(x) }
+        all_attribute_keys_array << :id
+
+        # If there is not version attribute, the version conditions will be ignored
+        version_attribute = if inventory_collection.parallel_safe? && supports_remote_data_timestamp?(all_attribute_keys)
+                              :resource_timestamp
+                            elsif inventory_collection.parallel_safe? && supports_remote_data_version?(all_attribute_keys)
+                              :resource_version
+                            end
+
+        update_query = update_query_beginning(all_attribute_keys_array)
+        update_query += update_query_reset_version_columns(version_attribute)
+        update_query += update_query_from_values(hashes, all_attribute_keys_array, connection)
+        update_query += update_query_version_conditions(version_attribute)
+        update_query += update_query_returning
+
+        #_log.debug("Building update query for #{inventory_collection} of size #{inventory_collection.size}...Complete")
+
+        update_query
+      end
+
+      private
+
+      def update_query_beginning(all_attribute_keys_array)
+        <<-SQL
+          UPDATE #{table_name}
+            SET
+              #{all_attribute_keys_array.map { |key| build_update_set_cols(key) }.join(",")}
+        SQL
+      end
+
+      def update_query_reset_version_columns(version_attribute)
+        if version_attribute
+          attr_partial     = version_attribute.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+          attr_partial_max = "#{attr_partial}_max"
+
+          # Quote the column names
+          attr_partial     = quote_column_name(attr_partial)
+          attr_partial_max = quote_column_name(attr_partial_max)
+
+          # Full row update will reset the partial update timestamps
+          <<-SQL
+            , #{attr_partial} = '{}', #{attr_partial_max} = NULL
+          SQL
+        else
+          ""
+        end
+      end
+
+      def update_query_from_values(hashes, all_attribute_keys_array, connection)
+        values = hashes.map! do |hash|
+          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x, true) }.join(",")})"
+        end.join(",")
+
+        <<-SQL
+          FROM (
+            VALUES
+              #{values}
+          ) AS updated_values (#{all_attribute_keys_array.map { |x| quote_column_name(x) }.join(",")})
+          WHERE updated_values.id = #{q_table_name}.id
+        SQL
+      end
+
+      def update_query_version_conditions(version_attribute)
+        if version_attribute
+          # This conditional will avoid rewriting new data by old data. But we want it only when version_attribute is
+          # a part of the data, since for the fake records, we just want to update ems_ref.
+          attr_partial     = version_attribute.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+          attr_partial_max = "#{attr_partial}_max"
+
+          # Quote the column names
+          attr_full        = quote_column_name(version_attribute)
+          attr_partial_max = quote_column_name(attr_partial_max)
+
+          <<-SQL
+            AND (
+              updated_values.#{attr_full} IS NULL OR (
+                (#{q_table_name}.#{attr_full} IS NULL OR updated_values.#{attr_full} > #{q_table_name}.#{attr_full}) AND
+                (#{q_table_name}.#{attr_partial_max} IS NULL OR updated_values.#{attr_full} >= #{q_table_name}.#{attr_partial_max})
+              )
+            )
+          SQL
+        else
+          ""
+        end
+      end
+
+      def update_query_returning
+        if inventory_collection.parallel_safe?
+          <<-SQL
+            RETURNING updated_values.#{quote_column_name("id")}, #{unique_index_columns.map { |x| "updated_values.#{quote_column_name(x)}" }.join(",")}
+          SQL
+        else
+          ""
+        end
+      end
+    end
+  end
+end

data/lib/inventory_refresh/save_collection/saver/sql_helper_upsert.rb

@@ -0,0 +1,196 @@
+module InventoryRefresh::SaveCollection
+  module Saver
+    module SqlHelperUpsert
+      # Builds ON CONFLICT UPDATE updating branch for one column identified by the passed key
+      #
+      # @param key [Symbol] key that is column name
+      # @return [String] SQL clause for upserting one column
+      def build_insert_set_cols(key)
+        "#{quote_column_name(key)} = EXCLUDED.#{quote_column_name(key)}"
+      end
+
+      # @param all_attribute_keys [Array<Symbol>] Array of all columns we will be saving into each table row
+      # @param hashes [Array<Hash>] data used for building a batch insert sql query
+      # @param mode [Symbol] Mode for saving, allowed values are [:full, :partial], :full is when we save all
+      #        columns of a row, :partial is when we save only few columns, so a partial row.
+      # @param on_conflict [Symbol, NilClass] defines behavior on conflict with unique index constraint, allowed values
+      #        are :do_update, :do_nothing, nil
+      def build_insert_query(all_attribute_keys, hashes, on_conflict: nil, mode:, column_name: nil)
+        #_log.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...")
+
+        # Cache the connection for the batch
+        connection = get_connection
+        # Ignore versioning columns that are set separately
+        ignore_cols = mode == :partial ? [:resource_timestamp, :resource_version] : []
+        # Make sure we don't send a primary_key for INSERT in any form, it could break PG sequencer
+        all_attribute_keys_array = all_attribute_keys.to_a - [primary_key.to_s, primary_key.to_sym] - ignore_cols
+
+        insert_query = insert_query_insert_values(hashes, all_attribute_keys_array, connection)
+        insert_query += insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
+        insert_query += insert_query_returning
+
+        #_log.debug("Building insert query for #{inventory_collection} of size #{inventory_collection.size}...Complete")
+
+        insert_query
+      end
+
+      private
+
+      def insert_query_insert_values(hashes, all_attribute_keys_array, connection)
+        values = hashes.map do |hash|
+          "(#{all_attribute_keys_array.map { |x| quote(connection, hash[x], x) }.join(",")})"
+        end.join(",")
+
+        col_names = all_attribute_keys_array.map { |x| quote_column_name(x) }.join(",")
+
+        <<-SQL
+          INSERT INTO #{q_table_name} (#{col_names})
+            VALUES
+              #{values}
+        SQL
+      end
+
+      def insert_query_on_conflict_behavior(all_attribute_keys, on_conflict, mode, ignore_cols, column_name)
+        return "" unless inventory_collection.parallel_safe?
+
+        insert_query_on_conflict = insert_query_on_conflict_do(on_conflict)
+        if on_conflict == :do_update
+          insert_query_on_conflict += insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
+        end
+        insert_query_on_conflict
+      end
+
+      def insert_query_on_conflict_do(on_conflict)
+        if on_conflict == :do_nothing
+          <<-SQL
+            ON CONFLICT DO NOTHING
+          SQL
+        elsif on_conflict == :do_update
+          index_where_condition = unique_index_for(unique_index_keys).where
+          where_to_sql          = index_where_condition ? "WHERE #{index_where_condition}" : ""
+
+          <<-SQL
+            ON CONFLICT (#{unique_index_columns.map { |x| quote_column_name(x) }.join(",")}) #{where_to_sql}
+              DO
+                UPDATE
+          SQL
+        end
+      end
+
+      def insert_query_on_conflict_update(all_attribute_keys, mode, ignore_cols, column_name)
+        if mode == :partial
+          ignore_cols += [:resource_timestamps, :resource_timestamps_max, :resource_versions, :resource_versions_max]
+        end
+        ignore_cols += [:created_on, :created_at] # Lets not change created for the update clause
+
+        # If there is not version attribute, the update part will be ignored below
+        version_attribute = if supports_remote_data_timestamp?(all_attribute_keys)
+                              :resource_timestamp
+                            elsif supports_remote_data_version?(all_attribute_keys)
+                              :resource_version
+                            end
+
+        # TODO(lsmola) should we add :deleted => false to the update clause? That should handle a reconnect, without a
+        # a need to list :deleted anywhere in the parser. We just need to check that a model has the :deleted attribute
+        query = <<-SQL
+          SET #{(all_attribute_keys - ignore_cols).map { |key| build_insert_set_cols(key) }.join(", ")}
+        SQL
+
+        # This conditional will make sure we are avoiding rewriting new data by old data. But we want it only when
+        # remote_data_timestamp is a part of the data.
+        query += insert_query_on_conflict_update_mode(mode, version_attribute, column_name) if version_attribute
+        query
+      end
+
+      def insert_query_on_conflict_update_mode(mode, version_attribute, column_name)
+        if mode == :full
+          full_update_condition(version_attribute)
+        elsif mode == :partial
+          raise "Column name must be provided" unless column_name
+          partial_update_condition(version_attribute, column_name)
+        end
+      end
+
+      def full_update_condition(attr_full)
+        attr_partial = attr_full.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+        attr_partial_max = "#{attr_partial}_max"
+
+        # Quote the column names
+        attr_full        = quote_column_name(attr_full)
+        attr_partial     = quote_column_name(attr_partial)
+        attr_partial_max = quote_column_name(attr_partial_max)
+
+        <<-SQL
+          , #{attr_partial} = '{}', #{attr_partial_max} = NULL
+
+          WHERE EXCLUDED.#{attr_full} IS NULL OR (
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_full} > #{q_table_name}.#{attr_full}) AND
+            (#{q_table_name}.#{attr_partial_max} IS NULL OR EXCLUDED.#{attr_full} >= #{q_table_name}.#{attr_partial_max})
+          )
+        SQL
+      end
+
+      def partial_update_condition(attr_full, column_name)
+        attr_partial     = attr_full.to_s.pluralize # Changes resource_version/timestamp to resource_versions/timestamps
+        attr_partial_max = "#{attr_partial}_max"
+        cast             = if attr_full == :resource_timestamp
+                             "timestamp"
+                           elsif attr_full == :resource_version
+                             "integer"
+                           end
+
+        # Quote the column names
+        attr_full        = quote_column_name(attr_full)
+        attr_partial     = quote_column_name(attr_partial)
+        attr_partial_max = quote_column_name(attr_partial_max)
+        column_name      = get_connection.quote_string(column_name.to_s)
+        q_table_name     = get_connection.quote_table_name(table_name)
+
+        <<-SQL
+          #{insert_query_set_jsonb_version(cast, attr_partial, attr_partial_max, column_name)}
+          , #{attr_partial_max} = greatest(#{q_table_name}.#{attr_partial_max}::#{cast}, EXCLUDED.#{attr_partial_max}::#{cast})
+          WHERE EXCLUDED.#{attr_partial_max} IS NULL OR (
+            (#{q_table_name}.#{attr_full} IS NULL OR EXCLUDED.#{attr_partial_max} > #{q_table_name}.#{attr_full}) AND (
+              (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast} IS NULL OR
+              EXCLUDED.#{attr_partial_max}::#{cast} > (#{q_table_name}.#{attr_partial}->>'#{column_name}')::#{cast}
+            )
+          )
+        SQL
+      end
+
+      def insert_query_set_jsonb_version(cast, attr_partial, attr_partial_max, column_name)
+        if cast == "integer"
+          # If we have integer value, we don't want to encapsulate the value in ""
+          <<-SQL
+            , #{attr_partial} = #{q_table_name}.#{attr_partial} || ('{"#{column_name}": ' || EXCLUDED.#{attr_partial_max}::#{cast} || '}')::jsonb
+          SQL
+        else
+          <<-SQL
+            , #{attr_partial} = #{q_table_name}.#{attr_partial} || ('{"#{column_name}": "' || EXCLUDED.#{attr_partial_max}::#{cast} || '"}')::jsonb
+          SQL
+        end
+      end
+
+      def insert_query_returning
+        <<-SQL
+          RETURNING "id",#{unique_index_columns.map { |x| quote_column_name(x) }.join(",")}
+                    #{insert_query_returning_timestamps}
+        SQL
+      end
+
+      def insert_query_returning_timestamps
+        if inventory_collection.parallel_safe?
+          # For upsert, we'll return also created and updated timestamps, so we can recognize what was created and what
+          # updated
+          if inventory_collection.internal_timestamp_columns.present?
+            <<-SQL
+              , #{inventory_collection.internal_timestamp_columns.map { |x| quote_column_name(x) }.join(",")}
+            SQL
+          end
+        else
+          ""
+        end
+      end
+    end
+  end
+end