etna 0.1.15 → 0.1.21

data/lib/etna/clients/magma/workflows/model_synchronization_workflow.rb

@@ -0,0 +1,306 @@
+require 'ostruct'
+
+module Etna
+  module Clients
+    class Magma
+      # Note!  These workflows are not perfectly atomic, nor perfectly synchronized due to nature of the backend.
+      # These primitives are best effort locally synchronized, but cannot defend the backend or simultaneous
+      # system updates.
+      class ModelSynchronizationWorkflow < Struct.new(
+          :target_client, :source_models, :target_project, :update_block,
+          :model_name, :plan_only, :validate, :use_versions, :renames,
+          keyword_init: true)
+        def target_models
+          @target_models ||= begin
+            target_client.retrieve(RetrievalRequest.new(project_name: self.target_project, model_name: 'all')).models
+          end
+        end
+
+        def planned_actions
+          @planned_actions ||= []
+        end
+
+        def queue_update(action)
+          if plan_only
+            plan_update(action)
+          else
+            execute_update(action)
+          end
+        end
+
+        def execute_planned!
+          @planned_actions.each do |action|
+            execute_update(action)
+          end
+        end
+
+        def execute_update(action)
+          update = UpdateModelRequest.new(project_name: self.target_project)
+          update_block.call(action) if update_block
+          update.add_action(action)
+          @target_models = nil
+          target_client.update_model(update)
+        end
+
+        def copy_link_into_target(link, reciprocal)
+          attr = target_models.build_model(link.model_name).build_template.build_attributes.build_attribute(link.attribute_name)
+          attr.attribute_type = link.type
+          attr.name = attr.attribute_name = link.attribute_name
+          attr.link_model_name = reciprocal.model_name
+        end
+
+        # Applies the given action to the source models and 'plans' its execution.
+        def plan_update(action)
+          case action
+          when UpdateAttributeAction
+            attribute_update = target_models.build_model(action.model_name).build_template.build_attributes.build_attribute(action.attribute_name)
+            Attribute.copy(action, attribute_update, attributes: Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES)
+          when AddAttributeAction
+            new_attribute = target_models.build_model(action.model_name).build_template.build_attributes.build_attribute(action.attribute_name)
+            Attribute.copy(action, new_attribute)
+          when AddLinkAction
+            first_link = action.links[0]
+            second_link = action.links[1]
+            copy_link_into_target(first_link, second_link)
+            copy_link_into_target(second_link, first_link)
+          when AddModelAction
+            template = target_models.build_model(action.model_name).build_template
+            template.name = action.model_name
+            template.identifier = action.identifier
+            template.parent = action.parent_model_name
+
+            child_link = AddLinkDefinition.new(type: AttributeType::PARENT, model_name: template.name, attribute_name: template.parent)
+            parent_link = AddLinkDefinition.new(type: action.parent_link_type, model_name: template.parent, attribute_name: template.name)
+
+            copy_link_into_target(child_link, parent_link)
+            copy_link_into_target(parent_link, child_link)
+
+            ['created_at', 'updated_at'].each do |time_attr_name|
+              template.build_attributes.build_attribute(time_attr_name).tap do |attr|
+                attr.attribute_type = Etna::Clients::Magma::AttributeType::DATE_TIME
+                attr.attribute_name = time_attr_name
+              end
+            end
+
+            if action.parent_link_type != Etna::Clients::Magma::AttributeType::TABLE
+              template.build_attributes.build_attribute(template.identifier).tap do |attr|
+                attr.attribute_name = template.identifier
+                attr.attribute_type = Etna::Clients::Magma::AttributeType::IDENTIFIER
+              end
+            end
+          when RenameAttributeAction
+            attributes = target_models.model(action.model_name).template.attributes
+            attributes.raw[action.new_attribute_name] = attributes.raw.delete(action.attribute_name)
+          else
+            raise "Unexpected plan_action #{action}"
+          end
+
+          planned_actions << action
+        end
+
+
+        def self.from_api_source(source_project:, source_client:, **kwds)
+          self.new(
+              source_models: source_client.retrieve(RetrievalRequest.new(project_name: source_project, model_name: 'all')).models,
+              **kwds
+          )
+        end
+
+        # Subclass and override when the source <-> target mapping is not 1:1.
+        def target_of_source(model_name)
+          model_name
+        end
+
+        # Subclass and override when the source <-> target attribute mapping is not 1:1.
+        def target_attribute_of_source(model_name, attribute_name)
+          attribute_name
+        end
+
+        # Potentially cyclical, protected against re-entry via the seen cache.
+        # Establishes the link attributes in a given model graph.
+        def ensure_model_tree(model_name, seen = Set.new)
+          return unless (source_model = source_models.model(model_name))
+          return if seen.include?(model_name)
+          seen.add(model_name)
+          ensure_model(model_name)
+
+          attributes = source_model.template.attributes
+
+          attributes.all.each do |attribute|
+            # Don't copy or update parent links.  Once a model has been setup with a parent someway.
+            unless attribute.attribute_type == AttributeType::PARENT
+              if attribute.link_model_name
+                ensure_model(attribute.link_model_name)
+                ensure_model_link(model_name, attribute.link_model_name, attribute.attribute_name)
+              else
+                ensure_model_attribute(model_name, attribute.attribute_name)
+              end
+            end
+
+            # Even if it's a parent node, however, we still want to cascade the tree expansion to all links.
+            unless attribute.link_model_name.nil?
+              ensure_model_tree(attribute.link_model_name, seen)
+            end
+          end
+        end
+
+        def ensure_model_link(model_name, link_model_name, attribute_name)
+          return unless (model = source_models.model(model_name))
+          return unless (source_attribute = model.template.attributes.attribute(attribute_name))
+
+          return unless (link_model = source_models.model(link_model_name))
+          link_model_attributes = link_model.template.attributes
+          reciprocal = link_model_attributes.all.find do |attr|
+            attr.link_model_name == model_name
+          end
+
+          target_model_name = target_of_source(model_name)
+          target_link_model_name = target_of_source(link_model_name)
+
+          target_attributes = target_models.model(target_model_name).template.attributes
+          return if target_attributes.attribute_keys.include?(target_link_model_name)
+
+          add_link = AddLinkAction.new
+          add_link.links << AddLinkDefinition.new(model_name: target_model_name, attribute_name: target_link_model_name, type: source_attribute.attribute_type)
+          add_link.links << AddLinkDefinition.new(model_name: target_link_model_name, attribute_name: reciprocal.attribute_name, type: reciprocal.attribute_type)
+
+          queue_update(add_link)
+        end
+
+        def ensure_model_attribute(model_name, attribute_name)
+          return unless (model = source_models.model(model_name))
+          return unless (source_attribute = model.template.attributes.attribute(attribute_name))
+
+          target_model_name = target_of_source(model_name)
+          target_attribute, target_attribute_name = ensure_model_attribute_target_rename(model_name, attribute_name)
+
+          if target_attribute.nil?
+            add_attribute = AddAttributeAction.new(
+                model_name: target_model_name,
+                attribute_name: target_attribute_name,
+            )
+
+            Attribute.copy(source_attribute, add_attribute)
+            queue_update(add_attribute)
+          else
+            # If there are is no diff, don't produce an action.
+            target_attribute = Attribute.new(target_attribute.raw)
+            target_attribute.set_field_defaults!
+
+            source_attribute = Attribute.new(source_attribute.raw)
+            source_attribute.set_field_defaults!
+
+            source_editable = source_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
+            target_editable = target_attribute.raw.slice(*Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES.map(&:to_s))
+
+            if source_editable == target_editable
+              return
+            end
+
+            update_attribute = UpdateAttributeAction.new(
+                model_name: target_model_name,
+                attribute_name: target_attribute_name,
+            )
+
+            Attribute.copy(source_attribute, update_attribute, attributes: Attribute::EDITABLE_ATTRIBUTE_ATTRIBUTES)
+            queue_update(update_attribute)
+          end
+        end
+
+        # Returns a tuple of the target's attribute, post rename if necessary, if it exists, and the name of the target attribute
+        # cases here:
+        # 1.  There is no rename for the attribute
+        #  a. There target attribute already exists -> [target_attribute, attribute_name]
+        #  b. The target attribute does not exist -> [nil, attribute_name]
+        # 2.  There is an expected rename from the source
+        #  a. The target has neither the renamed attribute or the original attribute -> [nil, new_attribute_name]
+        #  b. The target has the renamed attribute already -> [renamed_attribute, new_attribute_name]
+        #  c. The target has the source attribute, which is not yet renamed. -> [renamed_attribute, new_attribute_name]
+        def ensure_model_attribute_target_rename(model_name, attribute_name)
+          target_model_name = target_of_source(model_name)
+          target_attribute_name = target_attribute_of_source(model_name, attribute_name)
+          return nil unless (target_model = target_models.model(target_model_name))
+
+          target_original_attribute = target_model.template.attributes.attribute(target_attribute_name)
+
+          if renames && (attribute_renames = renames[model_name]) && (new_name = attribute_renames[attribute_name])
+            new_name = target_attribute_of_source(model_name, new_name)
+
+            unless target_model.template.attributes.include?(new_name)
+              if target_original_attribute
+                rename = RenameAttributeAction.new(model_name: target_model_name, attribute_name: target_attribute_name, new_attribute_name: new_name)
+                queue_update(rename)
+              else
+                return [nil, new_name]
+              end
+            end
+
+            return [target_model.template.attributes.attribute(new_name), new_name]
+          end
+
+          [target_original_attribute, target_attribute_name]
+        end
+
+        # Non cyclical, non re-entrant due to the requirement that parents cannot form a cycle.
+        # This method, and it's partner prepare_parent, should never call into any re-entrant or potentially
+        # cyclical method, like ensure_model_tree.
+        def ensure_model(model_name)
+          return unless (source_model = source_models.model(model_name))
+          target_model_name = target_of_source(model_name)
+          return if target_models.model_keys.include?(target_model_name)
+
+          template = source_model.template
+
+          add_model_action = AddModelAction.new(
+              {
+                  model_name: target_model_name,
+                  identifier: template.identifier,
+              }
+          )
+
+          parents = template.attributes.all.select { |a| a.attribute_type == AttributeType::PARENT }
+          parent_model_name, parent_link_type = prepare_parent(model_name, template, parents)
+          unless parent_model_name.nil?
+            add_model_action.parent_model_name = parent_model_name
+            add_model_action.parent_link_type = parent_link_type
+          end
+
+          queue_update(add_model_action)
+        end
+
+        # Non cyclical, non re-entrant due to the requirement that parents cannot form a cycle.
+        # This method, and it's partner and ensure_model, should never call into any re-entrant or potentially
+        # cyclical method, like ensure_model_tree.
+        def prepare_parent(model_name, template, parents)
+          return [nil, nil] if parents.empty?
+          return [nil, nil] unless (parent_model = source_models.model(template.parent))
+          return [nil, nil] unless (child_attribute = parent_model.template.attributes.attribute(model_name))
+
+          ensure_model(template.parent)
+
+          [
+              target_of_source(template.parent),
+              child_attribute.attribute_type
+          ]
+        end
+
+        def self.models_affected_by(action)
+          case action
+          when Etna::Clients::Magma::RenameAttributeAction
+            [action.model_name]
+          when Etna::Clients::Magma::UpdateAttributeAction
+            [action.model_name]
+          when Etna::Clients::Magma::AddAttributeAction
+            [action.model_name]
+          when Etna::Clients::Magma::AddModelAction
+            [action.model_name]
+          when Etna::Clients::Magma::AddLinkAction
+            action.links.map(&:model_name)
+          else
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/record_synchronization_workflow.rb

@@ -0,0 +1,63 @@
+require 'ostruct'
+
+module Etna
+  module Clients
+    class Magma
+      class RecordSynchronizationWorkflow < Struct.new(:target_client, :source_client, :project_name, :ignore_update_errors, keyword_init: true)
+        def target_models
+          @target_models ||= begin
+            target_client.retrieve(RetrievalRequest.new(project_name: self.project_name, model_name: 'all')).models
+          end
+        end
+
+        def crud
+          @crud ||= MagmaCrudWorkflow.new(project_name: project_name, magma_client: target_client)
+        end
+
+        def source_models
+          @source_models ||= source_client.retrieve(
+              RetrievalRequest.new(
+                  project_name: project_name, model_name: 'all')).models
+        end
+
+        # TODO: Add paging here to support large payloads
+        def copy_model(model_name, copied_models = Set.new)
+          if copied_models.include?(model_name)
+            return
+          end
+
+          copied_models.add(model_name)
+
+          descendants = source_models.to_directed_graph.descendants("project")
+          (descendants[model_name] || []).each do |required_model|
+            copy_model(required_model, copied_models) do |update|
+              yield update if block_given?
+            end
+          end
+
+          crud.page_records(model_name) do |documents|
+            yield [model_name, documents] if block_given?
+            begin
+              crud.update_records do |update_request|
+                documents.document_keys.each do |identifier|
+                  record = documents.document(identifier)
+                  record = record.each.select { |k, v| v != nil && !v.is_a?(Array) }.to_h
+                  update_request.update_revision(model_name, identifier, record)
+                end
+              end
+            rescue => e
+              raise unless ignore_update_errors
+            end
+          end
+        end
+
+        def copy_all_models
+          copied_models = Set.new
+          target_models.model_keys.each do |model_name|
+            copy_model(model_name, copied_models)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/update_attributes_from_csv_workflow.rb

@@ -0,0 +1,246 @@
+require 'csv'
+require 'ostruct'
+require_relative './crud_workflow'
+
+module Etna
+  module Clients
+    class Magma
+      class UpdateAttributesFromCsvWorkflowBase < Struct.new(:magma_crud, :project_name, :filepath, :model_name, :json_values, :hole_value, keyword_init: true)
+        def initialize(opts)
+          super(**{}.update(opts))
+        end
+
+        def magma_client
+          magma_crud.magma_client
+        end
+
+        def parse_input_file
+          raise "Must be implemented in a subclass"
+        end
+
+        def model_exists?(model_name)
+          models.model(model_name)
+        end
+
+        def consolidate_attributes_to_hash(row)
+          raise "Must be implemented in a subclass"
+        end
+
+        def find_attribute(model_name, attribute_name)
+          models.model(model_name).template.attributes.attribute(attribute_name)
+        end
+
+        def each_revision
+          parse_input_file do |model_name, record_name, revision|
+            yield [model_name, record_name, revision]
+          end
+        end
+
+        def update_attributes
+          method = json_values ? :update_json : :update
+          magma_crud.update_records(method: method) do |update_request|
+            each_revision do |model_name, record_name, revision|
+              update_request.update_revision(model_name, record_name, revision)
+            end
+          end
+        end
+
+        def models
+          @models ||= begin
+            magma_client.retrieve(RetrievalRequest.new(project_name: self.project_name, model_name: 'all')).models
+          end
+        end
+      end
+
+      class RowBase
+        def stripped_value(attribute_value)
+          attribute_value = attribute_value&.strip
+
+          if attribute_value && @workflow.json_values && attribute_value != @workflow.hole_value
+            attribute_value = JSON.parse(attribute_value)
+          end
+          attribute_value
+        end
+
+        def nil_or_empty?(value)
+          value.nil? || value.empty?
+        end
+      end
+
+      class UpdateAttributesFromCsvWorkflowMultiModel < UpdateAttributesFromCsvWorkflowBase
+        def parse_input_file
+          CSV.parse(File.read(filepath)).map do |csv_row|
+            row = Row.new(csv_row, self)
+
+            raise "Invalid model \"#{row.model_name}\" for project #{project_name}." unless model_exists?(row.model_name)
+
+            yield [row.model_name, row.record_name, row.to_h]
+          end
+        end
+
+        class Row < RowBase
+          attr_reader :model_name, :record_name
+
+          def initialize(raw, workflow)
+            # Assumes rows are in pairs, where
+            #   [0] = model_name
+            #   [1] = record_name / identifier
+            #   [2], [4], etc. = attribute_name
+            #   [3], [5], etc. = attribute value
+            # So not every row needs the same number of columns
+            @raw = raw
+            @workflow = workflow
+
+            raise "Invalid revision row #{@raw}. Must include at least 4 column values (model,record_name,attribute_name,attribute_value)." if @raw.length < 4
+            raise "Invalid revision row #{@raw}. Must have an even number of columns." if @raw.length.odd?
+
+            @model_name = raw[0]
+
+            raise "Invalid model name: \"#{@model_name}\"." if nil_or_empty?(@model_name)
+
+            @model_name.strip!
+
+            @record_name = raw[1]
+
+            raise "Invalid record name: \"#{@record_name}\"." if nil_or_empty?(@record_name)
+
+            @record_name.strip!
+          end
+
+          def to_h
+            # Take attribute index values (even) and put them into a hash.
+            # The assigned value will be the subsequent odd index.
+            # {attribute_name: attribute_value}
+            {}.tap do |attributes|
+              (2..(@raw.length - 1)).to_a.each do |index|
+                if index.even?
+                  attribute_name = @raw[index]
+
+                  raise "Invalid attribute name: \"#{attribute_name}\"." if nil_or_empty?(attribute_name)
+                  attribute_name.strip!
+
+                  unless (attribute = @workflow.find_attribute(model_name, attribute_name))
+                    raise "Invalid attribute #{attribute_name} for model #{model_name}."
+                  end
+
+                  stripped = stripped_value(@raw[index + 1])
+                  unless @workflow.hole_value.nil?
+                    next if stripped == @workflow.hole_value
+                  end
+
+                  if attribute.is_project_name_reference?(model_name)
+                    stripped&.downcase!
+                  end
+
+                  attributes[attribute_name] = stripped
+                end
+              end
+            end
+          end
+        end
+      end
+
+      class SimpleFileLinkingWorkflow < Struct.new(:metis_client, :project_name, :bucket_name, :folder, :extension, :attribute_name, :regex, :file_collection, keyword_init: true)
+        def write_csv_io(filename: nil, output_io: nil)
+          exporter = Etna::CsvExporter.new([:identifier, attribute_name.to_sym])
+          exporter.with_row_writeable(filename: filename, output_io: output_io) do |row_writeable|
+            find_matching_revisions.each do |identifier, value|
+              row_writeable << { identifier: identifier, attribute_name.to_sym => value.to_json }
+            end
+          end
+        end
+
+        def find_matching_revisions
+          {}.tap do |revisions|
+            metis_client.find(
+                Etna::Clients::Metis::FindRequest.new(
+                    project_name: project_name,
+                    bucket_name: bucket_name,
+                    params: [Etna::Clients::Metis::FindParam.new(
+                        attribute: 'name',
+                        predicate: 'glob',
+                        value: "#{folder}/**/*.#{extension}",
+                        type: 'file'
+                    )]
+                )).files.all.each do |file|
+              puts "Checking #{file.file_path}"
+              match = regex.match(file.file_path)
+              if match
+                match_map = match.names.zip(match.captures).to_h
+                if !match_map.include?('identifier')
+                  raise "Regex #{regex.source} does not include a ?<identifier> named matcher, please add one to regulate how identifiers are created."
+                end
+
+                puts "Found match"
+
+                revision = { 'path' => "metis://#{project_name}/#{bucket_name}/#{file.file_path}", 'original_filename' => "#{File.basename(file.file_path)}" }
+                if file_collection
+                  collection = revisions[match_map['identifier']] ||= []
+                  collection << revision
+                else
+                  record = revisions[match_map['identifier']] ||= {}
+                  unless record.empty?
+                    raise "Multiple files match #{match_map['identifier']}, found #{record['path']} and #{revision['path']}"
+                  end
+                  record.update(revision)
+                end
+              end
+            end
+          end
+        end
+      end
+
+      class UpdateAttributesFromCsvWorkflowSingleModel < UpdateAttributesFromCsvWorkflowBase
+        def initialize(opts)
+          super(**{}.update(opts))
+          raise "Single Model invocation must include keyword :model_name." if !opts[:model_name]
+          raise "Invalid model #{model_name} for project #{project_name}." unless model_exists?(model_name)
+        end
+
+        def parse_input_file
+          CSV.parse(File.read(filepath), headers: true).map do |csv_row|
+            row = Row.new(csv_row, model_name, self)
+
+            yield [model_name, row.record_name, row.to_h]
+          end
+        end
+
+        class Row < RowBase
+          attr_reader :record_name
+
+          def initialize(raw, model_name, workflow)
+            # Assumes CSV includes a column header to identify the attribute_name
+            # Assumes index 0 is the record_name
+            @raw = raw
+            @model_name = model_name
+            @workflow = workflow
+
+            @record_name = @raw[0]
+            raise "Invalid record name: \"#{record_name}\"." if nil_or_empty?(record_name)
+
+            @record_name.strip!
+          end
+
+          def to_h
+            # Row can be converted to a hash, where keys are attribute_names and the
+            #   values come from the CSV
+            # {attribute_name: attribute_value}
+            {}.tap do |attributes|
+              row_hash = @raw.to_h
+              row_keys = row_hash.keys
+              row_keys[1..row_keys.length - 1].each do |attribute_name|
+                raise "Invalid attribute name: \"#{attribute_name}\"." if nil_or_empty?(attribute_name)
+
+                attribute_name_clean = attribute_name.strip
+                raise "Invalid attribute \"#{attribute_name_clean}\" for model #{@model_name}." unless attribute = @workflow.find_attribute(@model_name, attribute_name_clean)
+
+                attributes[attribute_name_clean] = stripped_value(@raw[attribute_name])
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
+