extraction_metadata_changes 0.0.1a

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c7e5c61dc16159a4a8c989eb63689b147af5c70c35a3a96bb15d0d320374d914
+  data.tar.gz: bcf65732c185b576ccc4d15c10e6ffba1d2edab4d58cfa368433ea884858d973
+SHA512:
+  metadata.gz: 7d6f2b597f0a240f16e72eef7d16bb25fa8d37a7d894b1dcd12e16b23da55706c131ea900b215ec726fd30790b02cf3f41cf5ef99e76c0487c1ad9fe1d69bcff
+  data.tar.gz: 3670c2f5811142be4940514162cc9320aa2d2fd3a6e3379bba601fbc7bb712c383eb1af6d9e59bf4b8cb9b772de692a43b9ce6d31119f8643f1b7763bf0204f5

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Eduardo Martín Rojo
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+# extraction-metadata-changes
+Client interface tool that talks with the metadata service to store and apply all
+metadata modifications in a single transaction.
+
+# How to use it
+
+Add this line to your Gemfile:
+
+```ruby
+gem "extraction_metadata_changes"
+```
+
+# Getting started
+
+## Building a modifications object
+
+To build a set of metadata modifications, first we need to build a FactChanges object:
+
+```ruby
+> changes = FactChanges.new
+```
+
+This object will store the modifications that we want to apply in a single transaction so
+we can run different methods to create these modifications. The list of available
+modifications is:
+
+ * Create/Delete assets
+
+```ruby
+ > changes.create_assets(["00000000-0000-0000-0000"])
+ > changes.create_assets(["?my_car", "?my_previous_car", "?your_car"])
+ > changes.delete_assets(["00000000-0000-0000-0000"])
+```
+
+Created assets can be described by either an uuid or by using a *variable* notation. A variable is any string starting with '?' that will identify the created asset in subsequent modifications, so we can refer to it in a more meaningful way than with a uuid.
+
+ * Add/Remove properties
+
+```ruby
+ > changes.add("?my_car", "color", "Red")
+ > changes.remove_where("00000000-0000-0000-0000", "size", "Big")
+```
+
+ * Add/Remove relations
+
+``` ruby
+ > changes.add("?my_car", "quickerThan", "?your_car")
+ > changes.removeWhere("?my_car", "quickerThan", "?my_previous_car")
+ > changes.removeWhere("?my_previous_car", "quickerThan", "?my_car")
+ ```
+
+ * Create/Delete groups of assets
+
+```ruby
+ > changes.create_asset_groups(["?my_parking", "?a_parking_with_fees"])
+ > changes.delete_asset_groups(["00000000-0000-0000-0000"])
+```
+
+ * Add/Remove assets to/from groups
+
+```ruby
+ > changes.add_assets_to_group("?my_parking", ["?my_car"])
+ > changes.remove_assets_from_group("?a_parking_with_fees", ["?my_car"])
+```
+
+ * Specify errors that will avoid the transaction to apply
+
+```ruby
+ > changes.set_errors(["This set of modifications are wrong."])
+```
+
+## Applying the modifications
+
+Once we have completed all changes we want to apply in a single transaction, we can run
+the apply method:
+
+```ruby
+> changes.apply(transaction_id)
+```
+
+All modifications will be joined under a single transaction, so we need to provide a unique
+identifier of this set of modifications. One way of doing this is keeping a separate table that will keep track of the transactions creation an using the id of this table as the transaction id for the metadata service. This transaction id will be the reference of the changes we have apply so we can roll it back later if we ever need it.
+that we have apply.

data/lib/extraction_metadata_changes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+# Main module for the gem
+module ExtractionMetadataChanges
+end

data/lib/extraction_metadata_changes/disjoint_list.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'google_hash'
+
+module ExtractionMetadataChanges
+  # This class DisjointList allow us to establish a relation between a list and a one or
+  # more dependent lists where all lists behave as disjoint sets of elements where any
+  # element added to any of the list cannot exist in anyother list at the same time so all
+  # lists negate elements to each other.
+  #
+  class DisjointList
+    SEED_FOR_UNIQUE_IDS = Random.rand(1000)
+    MAX_DEEP_UNIQUE_ID = 3
+
+    include Enumerable
+
+    attr_accessor :location_for_unique_id
+    attr_accessor :disjoint_lists
+    attr_accessor :list
+    attr_reader :name
+
+    DISABLED_NAME = 'DISABLED'
+
+    def initialize(list)
+      @name = "object_id_#{object_id}"
+
+      # Replace with a normal hash if we want to stop using it
+      @location_for_unique_id = GoogleHashDenseLongToRuby.new
+
+      @list = []
+      @disjoint_lists = [self]
+      list.each { |o| add(o) }
+    end
+
+    def add_disjoint_list(disjoint_list)
+      disjoints = (disjoint_list.disjoint_lists - disjoint_lists)
+      disjoint_lists.concat(disjoints).uniq
+
+      disjoints.each do |disjoint|
+        _synchronize_with_list(disjoint)
+
+        disjoint.location_for_unique_id = location_for_unique_id
+        disjoint.disjoint_lists = disjoint_lists
+      end
+    end
+
+    def store_for(element)
+      _store_for(unique_id_for_element(element))
+    end
+
+    def _store_for(unique_id)
+      store_name = location_for_unique_id[unique_id]
+      return nil if store_name.nil? || store_name == DISABLED_NAME
+
+      @disjoint_lists.select { |l| l.name == store_name }.first
+    end
+
+    def enabled?(element)
+      !store_for(element).nil?
+    end
+
+    def disabled?(element)
+      disabled_key?(unique_id_for_element(element))
+    end
+
+    def disabled_key?(key)
+      @location_for_unique_id[key] == DISABLED_NAME
+    end
+
+    def enabled_in_other_list?(element)
+      enabled?(element) && !include?(element)
+    end
+
+    def include?(element)
+      include_key?(unique_id_for_element(element))
+    end
+
+    def include_key?(key)
+      @location_for_unique_id[key] == name
+    end
+
+    def remove(element)
+      unique_id = unique_id_for_element(element)
+      remove_from_raw_list_by_id(unique_id)
+      @location_for_unique_id.delete(unique_id)
+    end
+
+    def remove_from_raw_list_by_id(unique_id)
+      @list.delete_if do |a|
+        unique_id_for_element(a) == unique_id
+      end
+    end
+
+    def length
+      @list.length
+    end
+
+    def each(&block)
+      @list.each(&block)
+    end
+
+    def [](index)
+      @list[index]
+    end
+
+    def flatten
+      @list.flatten
+    end
+
+    def uniq!
+      @list.uniq!
+    end
+
+    def <<(element)
+      if element.is_a?(Array)
+        element.each { |e| add(e) }
+      else
+        add(element)
+      end
+    end
+
+    def concat(element)
+      if element.is_a?(Array)
+        element.each { |e| add(e) }
+      else
+        add(element)
+      end
+    end
+
+    def push(element)
+      add(element)
+    end
+
+    def add(element)
+      return concat_disjoint_list(element) if element.is_a?(
+        ExtractionMetadataChanges::DisjointList
+      )
+
+      if enabled_in_other_list?(element)
+        disable(element)
+      elsif include?(element)
+        # It is already in our list, so we do not add it again
+        return false
+      else
+        enable(element)
+      end
+      self
+    end
+
+    def sum_function_for(value)
+      value.hash
+      # Value to create checksum and seed
+      # XXhash.xxh32(value, SEED_FOR_UNIQUE_IDS)
+    end
+
+    def unique_id_for_element(element)
+      _unique_id_for_element(element, 0)
+    end
+
+    def concat_disjoint_list(disjoint_list)
+      disjoint_list.location_for_unique_id.keys.each do |key|
+        _disable(key) if disjoint_list.disabled_key?(key)
+      end
+      disjoint_list.to_a.each { |val| add(val) }
+      self
+    end
+
+    def merge(disjoint_list)
+      disjoint_list.location_for_unique_id.keys.each do |key|
+        _disable(key) if !disjoint_list.include_key?(key) || disjoint_list.disabled_key?(key)
+      end
+      disjoint_list.to_a.each { |val| add(val) }
+      self
+    end
+
+    def enable(element)
+      return if disabled?(element)
+
+      unique_id = unique_id_for_element(element)
+      # Is not in any of the lists so we can add it
+      if element.is_a?(Enumerable) && !element.is_a?(Hash)
+        @list.concat(element)
+      else
+        @list.push(element)
+      end
+      @location_for_unique_id[unique_id] = name
+    end
+
+    def disable(element)
+      _disable(unique_id_for_element(element))
+    end
+
+    protected
+
+    def _synchronize_with_list(disjoint_list)
+      disjoint_list.location_for_unique_id.keys.each do |key|
+        unless location_for_unique_id[key] == DISABLED_NAME
+          # If my disjoint lists do not have the element
+          if location_for_unique_id[key].nil?
+            location_for_unique_id[key] = disjoint_list.location_for_unique_id[key]
+            _disable(key) if location_for_unique_id[key] == DISABLED_NAME
+          elsif location_for_unique_id[key] != disjoint_list.location_for_unique_id[key]
+            # If my lists have the element alredy
+            _disable(key)
+          end
+        end
+      end
+    end
+
+    def _disable(unique_id)
+      store = _store_for(unique_id)
+      store&.remove_from_raw_list_by_id(unique_id)
+      location_for_unique_id[unique_id] = DISABLED_NAME
+    end
+
+    def _unique_id_for_element(element, deep = 0)
+      return sum_function_for(SecureRandom.uuid) if deep == MAX_DEEP_UNIQUE_ID
+
+      if element.is_a?(String)
+        sum_function_for(element)
+      elsif element.respond_to?(:uuid) && !element.uuid.nil?
+        sum_function_for(element.uuid)
+      elsif element.respond_to?(:id) && !element.id.nil?
+        sum_function_for("#{element.class}_#{element.id}")
+      elsif element.is_a?(Hash)
+        if element.key?(:uuid) && !element[:uuid].nil?
+          sum_function_for(element[:uuid])
+        elsif element.key?(:predicate)
+          _unique_id_for_fact(element)
+        else
+          sum_function_for(element.keys.dup.concat(element.values.map do |val|
+            _unique_id_for_element(val, deep + 1)
+          end).join(''))
+        end
+      elsif element.is_a?(Enumerable)
+        sum_function_for(element.map { |o| _unique_id_for_element(o, deep + 1) }.join(''))
+      else
+        sum_function_for(element.to_s)
+      end
+    end
+
+    def _unique_id_for_fact(element)
+      sum_function_for([
+        (element[:asset_id] || element[:asset].id || element[:asset].object_id),
+        element[:predicate],
+        (element[:object] || element[:object_asset_id] || element[:object_asset].id ||
+          element[:object_asset].object_id)
+      ].join('_'))
+    end
+  end
+end

data/lib/extraction_metadata_changes/fact_changes.rb

@@ -0,0 +1,652 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'extraction_token_util'
+
+module ExtractionMetadataChanges
+  #
+  # Any change of metadata is composed of small modifications related with each other
+  # that can be applied in a transactional way so they can be cancelled, rolledback, etc.
+  #
+  # *About DisjointList*
+  #
+  # To be able to merge different configurations of changes, we need a way to keep track
+  # on the properties and values that have been added and removed to keep the modifications
+  # consistent, to avoid performing the operation twice, or in a wrong way.
+  #
+  # For instance, in these operations:
+  #
+  # > changes.add('?my_car', 'color', 'Red')
+  # > changes.remove_where('?my_car', 'color', 'Red')
+  #
+  # This modifications have opposite meaning so they do not apply at all, but if we add this:
+  #
+  # > changes.add('?my_car', 'color', 'Bright')
+  #
+  # It can apply because it refers to a different value. All this logic about when a property
+  # should not be part of the final transaction is performed using the DisjointList class,
+  # by establishing a disjoint relations between opposite lists (added properties,
+  # removed properties, etc...)
+  #
+  #
+  class FactChanges
+    attr_accessor :facts_to_destroy, :facts_to_add, :assets_to_create, :assets_to_destroy,
+                  :assets_to_add, :assets_to_remove, :wildcards, :instances_from_uuid,
+                  :asset_groups_to_create, :asset_groups_to_destroy, :errors_added,
+                  :already_added_to_list, :instances_by_unique_id,
+                  :facts_to_set_to_remote
+
+    attr_accessor :operations
+
+    def initialize(json = nil)
+      @assets_updated = []
+      reset
+      parse_json(json) if json
+    end
+
+    def parsing_valid?
+      @parsing_valid
+    end
+
+    def reset
+      @parsing_valid = false
+      @errors_added = []
+
+      @facts_to_set_to_remote = []
+
+      build_disjoint_lists(:facts_to_add, :facts_to_destroy)
+      build_disjoint_lists(:assets_to_create, :assets_to_destroy)
+      build_disjoint_lists(:asset_groups_to_create, :asset_groups_to_destroy)
+      build_disjoint_lists(:assets_to_add, :assets_to_remove)
+
+      @instances_from_uuid = GoogleHashDenseRubyToRuby.new
+      @wildcards = GoogleHashDenseRubyToRuby.new
+    end
+
+    def build_disjoint_lists(list, opposite)
+      list1 = ExtractionMetadataChanges::DisjointList.new([])
+      list2 = ExtractionMetadataChanges::DisjointList.new([])
+
+      list1.add_disjoint_list(list2)
+
+      send("#{list}=", list1)
+      send("#{opposite}=", list2)
+    end
+
+    def asset_group_asset_to_h(asset_group_asset_str)
+      obj = asset_group_asset_str.each_with_object({}) do |o, memo|
+        key = o[:asset_group]&.uuid || nil
+        memo[key] = [] unless memo[key]
+        memo[key].push(o[:asset].uuid)
+      end
+      obj.map do |k, v|
+        [k, v]
+      end
+    end
+
+    def to_h
+      {
+        'set_errors': @errors_added,
+        'create_assets': @assets_to_create.map(&:uuid),
+        'create_asset_groups': @asset_groups_to_create.map(&:uuid),
+        'delete_asset_groups': @asset_groups_to_destroy.map(&:uuid),
+        'delete_assets': @assets_to_destroy.map(&:uuid),
+        'add_facts': @facts_to_add.map do |f|
+          [
+            f[:asset].nil? ? nil : f[:asset].uuid,
+            f[:predicate],
+            (f[:object] || f[:object_asset].uuid)
+          ]
+        end,
+        'remove_facts': @facts_to_destroy.map do |f|
+          if f[:id]
+            fact = Fact.find(f[:id])
+            [fact.asset.uuid, fact.predicate, fact.object_value_or_uuid]
+          else
+            [
+              f[:asset].nil? ? nil : f[:asset].uuid,
+              f[:predicate],
+              (f[:object] || f[:object_asset].uuid)
+            ]
+          end
+        end,
+        'add_assets': asset_group_asset_to_h(@assets_to_add),
+        'remove_assets': asset_group_asset_to_h(@assets_to_remove)
+      }.reject { |_k, v| v.empty? }
+    end
+
+    def to_json(*_args)
+      JSON.pretty_generate(to_h)
+    end
+
+    def parse_json(json)
+      obj = JSON.parse(json)
+      %w[set_errors create_assets create_asset_groups delete_asset_groups
+         remove_facts add_facts delete_assets add_assets remove_assets].each do |action_type|
+        send(action_type, obj[action_type]) if obj[action_type]
+      end
+      @parsing_valid = true
+    end
+
+    def values_for_predicate(asset, predicate)
+      actual_values = asset.facts.with_predicate(predicate).map(&:object)
+      values_to_add = facts_to_add.map do |f|
+        f[:object] if (f[:asset] == asset) && (f[:predicate] == predicate)
+      end.compact
+      values_to_destroy = facts_to_destroy.map do |f|
+        f[:object] if (f[:asset] == asset) && (f[:predicate] == predicate)
+      end.compact
+      (actual_values + values_to_add - values_to_destroy)
+    end
+
+    def _build_fact_attributes(asset, predicate, object, options = {})
+      t = [asset, predicate, object, options]
+      params = { asset: t[0], predicate: t[1], literal: !t[2].is_a?(Asset) }
+      params[:literal] ? params[:object] = t[2] : params[:object_asset] = t[2]
+      params = params.merge(t[3]) if t[3]
+      params
+    end
+
+    def add(asset, predicate, object, options = {})
+      asset = find_asset(asset)
+      object = options[:literal] == true ? literal_token(object) : find_asset(object)
+
+      fact = _build_fact_attributes(asset, predicate, object, options)
+
+      facts_to_add << fact if fact
+    end
+
+    def literal_token(str)
+      ExtractionTokenUtil.quote_if_uuid(str)
+    end
+
+    def add_facts(lists)
+      lists.each { |list| add(list[0], list[1], list[2]) }
+      self
+    end
+
+    def remove_facts(lists)
+      lists.each { |list| remove_where(list[0], list[1], list[2]) }
+      self
+    end
+
+    def add_remote(asset, predicate, object, options = {})
+      return unless asset && predicate && object
+
+      add(asset, predicate, object, options.merge({ is_remote?: true }))
+    end
+
+    def replace_remote_relation(asset, predicate, object_asset, options = {})
+      replace_remote(asset, predicate, object_asset, options.merge({ literal: false }))
+    end
+
+    def replace_remote_property(asset, predicate, value, options = {})
+      replace_remote(asset, predicate, value, options.merge({ literal: true }))
+    end
+
+    def replace_remote(asset, predicate, object, options = {})
+      return unless asset && predicate && object
+
+      asset.facts.with_predicate(predicate).each do |fact|
+        # The value is updated from the remote instance so we remove the previous value
+        remove(fact)
+        # In any case they will be set as Remote, even if they are not removed in this update
+        facts_to_set_to_remote << fact
+      end
+      add_remote(asset, predicate, object, options)
+    end
+
+    def remove(fact)
+      return if fact.nil?
+
+      if fact.is_a?(Enumerable)
+        facts_to_destroy << fact.map { |o| o.attributes.symbolize_keys }
+      elsif fact.is_a?(Fact)
+        facts_to_destroy << fact.attributes.symbolize_keys if fact
+      end
+    end
+
+    def remove_where(subject, predicate, object)
+      subject = find_asset(subject)
+      object = find_asset(object)
+
+      fact = _build_fact_attributes(subject, predicate, object)
+
+      facts_to_destroy << fact if fact
+    end
+
+    def errors?
+      to_h.key?(:set_errors)
+    end
+
+    def merge_hash(hash1, hash2)
+      hash2.keys.each do |k|
+        hash1[k] = hash2[k]
+      end
+      hash1
+    end
+
+    def merge(fact_changes)
+      if fact_changes
+        # To keep track of already added object after merging with another fact changes object
+        # _add_already_added_from_other_object(fact_changes)
+        errors_added.concat(fact_changes.errors_added)
+        asset_groups_to_create.concat(fact_changes.asset_groups_to_create)
+        assets_to_create.concat(fact_changes.assets_to_create)
+        facts_to_add.concat(fact_changes.facts_to_add)
+        assets_to_add.concat(fact_changes.assets_to_add)
+        assets_to_remove.concat(fact_changes.assets_to_remove)
+        facts_to_destroy.concat(fact_changes.facts_to_destroy)
+        assets_to_destroy.concat(fact_changes.assets_to_destroy)
+        asset_groups_to_destroy.concat(fact_changes.asset_groups_to_destroy)
+        merge_hash(instances_from_uuid, fact_changes.instances_from_uuid)
+        merge_hash(wildcards, fact_changes.wildcards)
+      end
+      self
+    end
+
+    def apply(step, with_operations = true)
+      _handle_errors(step) unless errors_added.empty?
+      ActiveRecord::Base.transaction do |_t|
+        # We need step to have an allocated id to be able to link it with the operations
+        # so we have to create a new record if is not already stored
+        step.save unless step.persisted?
+
+        # Callbacks execution
+        _on_apply(step) if respond_to?(:_on_apply)
+
+        _set_remote_facts(facts_to_set_to_remote)
+
+        # Creates the facts and generate from it the list of operations
+        operations = [
+          _create_asset_groups(step, asset_groups_to_create, with_operations),
+          _create_assets(step, assets_to_create, with_operations),
+          _add_assets(step, assets_to_add, with_operations),
+          _remove_assets(step, assets_to_remove, with_operations),
+          _remove_facts(step, facts_to_destroy, with_operations),
+          _detach_assets(step, assets_to_destroy, with_operations),
+          _detach_asset_groups(step, asset_groups_to_destroy, with_operations),
+          _create_facts(step, facts_to_add, with_operations)
+        ].flatten.compact
+
+        # Writes all operations in a single call
+        unless operations.empty?
+          Operation.import(operations)
+          @operations = operations
+        end
+        step.save if step.changed?
+        _handle_errors(step) unless errors_added.empty?
+        reset
+      end
+    end
+
+    def assets_updated
+      return [] unless @operations
+
+      @assets_updated = Asset.where(id: @operations.pluck(:asset_id).uniq).distinct
+    end
+
+    def assets_for_printing
+      return [] unless @operations
+
+      asset_ids = @operations.select do |operation|
+        (operation.action_type == 'createAssets')
+      end.pluck(:object).uniq
+
+      ready_for_print_ids = @operations.select do |operation|
+        ((operation.action_type == 'addFacts') &&
+        (operation.predicate == 'is') &&
+        (operation.object == 'readyForPrint'))
+      end.map(&:asset).compact.uniq.map(&:uuid)
+
+      ids_for_print = asset_ids.concat(ready_for_print_ids).flatten.uniq
+      @assets_for_printing = Asset.for_printing.where(uuid: ids_for_print)
+    end
+
+    def find_asset(asset_or_uuid)
+      find_instance_of_class_by_uuid(Asset, asset_or_uuid)
+    end
+
+    def find_asset_group(asset_group_or_id)
+      find_instance_of_class_by_uuid(AssetGroup, asset_group_or_id)
+    end
+
+    def find_assets(assets_or_uuids)
+      assets_or_uuids.uniq.map do |asset_or_uuid|
+        find_instance_of_class_by_uuid(Asset, asset_or_uuid)
+      end
+    end
+
+    def build_assets(assets)
+      assets.uniq.map do |asset_or_uuid|
+        find_instance_of_class_by_uuid(Asset, asset_or_uuid, true)
+      end
+    end
+
+    def find_asset_groups(asset_groups_or_uuids)
+      asset_groups_or_uuids.uniq.map do |asset_group_or_uuid|
+        find_instance_of_class_by_uuid(AssetGroup, asset_group_or_uuid)
+      end
+    end
+
+    def build_asset_groups(asset_groups)
+      asset_groups.uniq.map do |asset_group_or_uuid|
+        find_instance_of_class_by_uuid(AssetGroup, asset_group_or_uuid, true)
+      end
+    end
+
+    def new_record?(uuid)
+      (instances_from_uuid[uuid] && instances_from_uuid[uuid].new_record?) == true
+    end
+
+    def find_instance_of_class_by_uuid(klass, instance_or_uuid_or_id, create = false)
+      if ExtractionTokenUtil.wildcard?(instance_or_uuid_or_id)
+        uuid = uuid_for_wildcard(instance_or_uuid_or_id)
+        # Do not try to find it if it is a new wildcard created
+        found = find_instance_from_uuid(klass, uuid) unless create
+        found = ((instances_from_uuid[uuid] ||= klass.new(uuid: uuid))) if !found && create
+      elsif ExtractionTokenUtil.uuid?(instance_or_uuid_or_id)
+        found = find_instance_from_uuid(klass, instance_or_uuid_or_id)
+        if !found && create
+          found = ((instances_from_uuid[instance_or_uuid_or_id] ||= klass.new(
+            uuid: instance_or_uuid_or_id
+          )))
+        end
+      else
+        found = instance_or_uuid_or_id
+      end
+      unless found
+        _produce_error([%(
+          Element identified by #{instance_or_uuid_or_id} should be declared before using it
+        )])
+      end
+      found
+    end
+
+    def uuid_for_wildcard(wildcard)
+      wildcards[wildcard] ||= SecureRandom.uuid
+    end
+
+    def wildcard_for_uuid(uuid)
+      wildcards.keys.select { |key| wildcards[key] == uuid }.first
+    end
+
+    def find_instance_from_uuid(klass, uuid)
+      found = klass.find_by(uuid: uuid) unless new_record?(uuid)
+      return found if found
+
+      instances_from_uuid[uuid]
+    end
+
+    def validate_instances(instances)
+      if instances.is_a?(Array)
+        instances.each { |a| raise StandardError, a if a.nil? }
+      else
+        raise StandardError, a if instances.nil?
+      end
+      instances
+    end
+
+    def set_errors(errors)
+      errors_added.concat(errors)
+      self
+    end
+
+    def create_assets(assets)
+      assets_to_create << validate_instances(build_assets(assets))
+      # assets_to_create.concat(validate_instances(build_assets(assets)))
+      self
+    end
+
+    def create_asset_groups(asset_groups)
+      asset_groups_to_create << validate_instances(build_asset_groups(asset_groups))
+      self
+    end
+
+    def delete_asset_groups(asset_groups)
+      asset_groups_to_destroy << validate_instances(find_asset_groups(asset_groups))
+      self
+    end
+
+    def delete_assets(assets)
+      assets_to_destroy << validate_instances(find_assets(assets))
+      self
+    end
+
+    def add_assets_to_group(group, assets)
+      add_assets([[group, assets]])
+    end
+
+    def remove_assets_from_group(group, assets)
+      remove_assets([[group, assets]])
+    end
+
+    def add_assets(list)
+      list.each do |elem|
+        if !elem.empty? && elem[1].is_a?(Array)
+          asset_group = elem[0].nil? ? nil : validate_instances(find_asset_group(elem[0]))
+          asset_ids = elem[1]
+        else
+          asset_group = nil
+          asset_ids = elem
+        end
+        assets = validate_instances(find_assets(asset_ids))
+        assets_to_add << assets.map { |asset| { asset_group: asset_group, asset: asset } }
+      end
+      self
+    end
+
+    def remove_assets(list)
+      list.each do |elem|
+        if !elem.empty? && elem[1].is_a?(Array)
+          asset_group = elem[0].nil? ? nil : validate_instances(find_asset_group(elem[0]))
+          asset_ids = elem[1]
+        else
+          asset_group = nil
+          asset_ids = elem
+        end
+        assets = validate_instances(find_assets(asset_ids))
+        assets_to_remove << assets.map { |asset| { asset_group: asset_group, asset: asset } }
+      end
+      self
+    end
+
+    private
+
+    def _handle_errors(step)
+      step.set_errors(errors_added)
+      _produce_error(errors_added) unless errors_added.empty?
+    end
+
+    def _produce_error(errors_added)
+      raise StandardError.new(message: errors_added.join("\n"))
+    end
+
+    def _set_remote_facts(facts)
+      Fact.where(id: facts.map(&:id).uniq.compact).update_all(is_remote?: true)
+    end
+
+    def _add_assets(step, asset_group_assets, with_operations = true)
+      modified_list = asset_group_assets.map do |o|
+        # If is nil, it will use the asset group from the step
+        o[:asset_group] = o[:asset_group] || step.asset_group
+        o
+      end
+      _instance_builder_for_import(AssetGroupsAsset, modified_list) do |instances|
+        _asset_group_operations('addAssets', step, instances) if with_operations
+      end
+    end
+
+    def _remove_assets(step, assets_to_remove, with_operations = true)
+      modified_list = assets_to_remove.map do |obj|
+        AssetGroupsAsset.where(
+          asset_group: obj[:asset_group] || step.asset_group,
+          asset: obj[:asset]
+        )
+      end
+      _instances_deletion(AssetGroupsAsset, modified_list) do |asset_group_assets|
+        _asset_group_operations('removeAssets', step, asset_group_assets) if with_operations
+      end
+    end
+
+    def _create_assets(step, assets, with_operations = true)
+      return unless assets
+
+      count = Asset.count + 1
+      assets = assets.each_with_index.map do |asset, barcode_index|
+        _build_barcode(asset, count + barcode_index)
+        asset
+      end
+      _instance_builder_for_import(Asset, assets) do |_instances|
+        _asset_operations('createAssets', step, assets) if with_operations
+      end
+    end
+
+    ## TODO:
+    # Possibly it could be moved to Asset before_save callback
+    #
+    def _build_barcode(asset, num)
+      barcode_type = values_for_predicate(asset, 'barcodeType').first
+
+      return if barcode_type == 'NoBarcode'
+
+      barcode = values_for_predicate(asset, 'barcode').first
+      if barcode
+        asset.barcode = barcode
+      else
+        asset.build_barcode(num)
+      end
+    end
+
+    def _detach_assets(step, assets, with_operations = true)
+      operations = _asset_operations('deleteAssets', step, assets) if with_operations
+      _instances_deletion(Fact, assets.map(&:facts).flatten.compact)
+      _instances_deletion(AssetGroupsAsset, assets.map(&:asset_groups_assets).flatten.compact)
+      operations
+    end
+
+    def _create_asset_groups(step, asset_groups, with_operations = true)
+      return unless asset_groups
+
+      asset_groups.each_with_index do |asset_group, _index|
+        asset_group.update_attributes(
+          name: ExtractionTokenUtil.to_asset_group_name(wildcard_for_uuid(asset_group.uuid)),
+          activity_owner: step.activity
+        )
+        asset_group.save
+      end
+      _asset_group_building_operations('createAssetGroups', step, asset_groups) if with_operations
+    end
+
+    def _detach_asset_groups(step, asset_groups, with_operations = true)
+      if with_operations
+        operations = _asset_group_building_operations('deleteAssetGroups', step, asset_groups)
+      end
+      instances = asset_groups.flatten
+      ids_to_remove = instances.map(&:id).compact.uniq
+
+      if ids_to_remove && !ids_to_remove.empty?
+        AssetGroup.where(id: ids_to_remove).update_all(activity_owner_id: nil)
+      end
+      operations
+    end
+
+    def _create_facts(step, params_for_facts, with_operations = true)
+      _instance_builder_for_import(Fact, params_for_facts) do |facts|
+        _fact_operations('addFacts', step, facts) if with_operations
+      end
+    end
+
+    def _remove_facts(step, facts_to_remove, with_operations = true)
+      ids = []
+      modified_list = facts_to_remove.each_with_object([]) do |data, memo|
+        if data[:id]
+          ids.push(data[:id])
+        elsif data[:object].is_a? String
+          elems = Fact.where(asset: data[:asset], predicate: data[:predicate],
+                             object: data[:object])
+        else
+          elems = Fact.where(asset: data[:asset], predicate: data[:predicate],
+                             object_asset: data[:object_asset])
+        end
+        memo.concat(elems) if elems
+      end.concat(Fact.where(id: ids))
+      _instances_deletion(Fact, modified_list) do
+        _fact_operations('removeFacts', step, modified_list) if with_operations
+      end
+    end
+
+    def _asset_group_building_operations(action_type, step, asset_groups)
+      asset_groups.map do |asset_group|
+        Operation.new(action_type: action_type, step: step, object: asset_group.uuid)
+      end
+    end
+
+    def _asset_group_operations(action_type, step, asset_group_assets)
+      asset_group_assets.map do |asset_group_asset, _index|
+        Operation.new(action_type: action_type, step: step,
+                      asset: asset_group_asset.asset, object: asset_group_asset.asset_group.uuid)
+      end
+    end
+
+    def _asset_operations(action_type, step, assets)
+      assets.map do |asset, _index|
+        # refer = (action_type == 'deleteAsset' ? nil : asset)
+        Operation.new(action_type: action_type, step: step, object: asset.uuid)
+      end
+    end
+
+    def listening_to_predicate?(predicate)
+      predicate == 'parent'
+    end
+
+    def _fact_operations(action_type, step, facts)
+      modified_assets = []
+      operations = facts.map do |fact|
+        modified_assets.push(fact.object_asset) if listening_to_predicate?(fact.predicate)
+        Operation.new(action_type: action_type, step: step,
+                      asset: fact.asset, predicate: fact.predicate, object: fact.object,
+                      object_asset: fact.object_asset)
+      end
+      modified_assets.flatten.compact.uniq.each(&:touch)
+      operations
+    end
+
+    def all_values_are_new_records(hash)
+      hash.values.all? do |value|
+        (value.respond_to?(:new_record?) && value.new_record?)
+      end
+    end
+
+    def _instance_builder_for_import(klass, params_list)
+      instances = params_list.map do |params_for_instance|
+        if params_for_instance.is_a?(klass)
+          params_for_instance if params_for_instance.new_record?
+        elsif all_values_are_new_records(params_for_instance) ||
+              !klass.exists?(params_for_instance)
+          klass.new(params_for_instance)
+        end
+      end.compact.uniq
+      instances.each do |instance|
+        instance.run_callbacks(:save) { false }
+        instance.run_callbacks(:create) { false }
+      end
+      return unless instances && !instances.empty?
+
+      klass.import(instances)
+      # import does not return the ids for the instances, so we need to reload
+      # again. Uuid is the only identificable attribute set
+      klass.synchronize(instances, [:uuid]) if klass == Asset
+      yield instances
+    end
+
+    def _instances_deletion(klass, instances)
+      operations = block_given? ? yield(instances) : instances
+      instances = instances.flatten
+      ids_to_remove = instances.map(&:id).compact.uniq
+
+      klass.where(id: ids_to_remove).delete_all if ids_to_remove && !ids_to_remove.empty?
+      operations
+    end
+  end
+end

metadata

@@ -0,0 +1,132 @@
+--- !ruby/object:Gem::Specification
+name: extraction_metadata_changes
+version: !ruby/object:Gem::Version
+  version: 0.0.1a
+platform: ruby
+authors:
+- Eduardo Martin Rojo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-02-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: extraction_token_util
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3a11
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.0.3a11
+- !ruby/object:Gem::Dependency
+  name: google_hash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: factory_bot
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.80'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.80'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.38'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.38'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/extraction_metadata_changes.rb
+- lib/extraction_metadata_changes/disjoint_list.rb
+- lib/extraction_metadata_changes/fact_changes.rb
+homepage: https://rubygems.org/gems/extraction_metadata_changes
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Client interface tool that talks with the metadata service to store and apply
+  all metadata modifications in a single transaction
+test_files: []