etna 0.1.16 → 0.1.18

data/lib/etna/clients/magma/workflows/file_linking_workflow.rb

@@ -0,0 +1,115 @@
+require 'ostruct'
+require_relative './crud_workflow'
+
+module Etna
+  module Clients
+    class Magma
+      class FileLinkingWorkflow < Struct.new(:magma_crud, :model_name, :metis_client, :project_name, :bucket_name, :matching_expressions, :attribute_options, keyword_init: true)
+        PATIENT_TIMEPOINT_REGEX = /([^-]+-[^-]+)-(DN?[0-9]+).*/
+
+        def initialize(opts)
+          super(**{attribute_options: {}, matching_expressions: []}.update(opts))
+        end
+
+        def magma_client
+          magma_crud.magma_client
+        end
+
+        def find_matches
+          {}.tap do |all_matches|
+            metis_client.folders(
+                project_name: project_name,
+                bucket_name: bucket_name
+            ).each do |folder|
+              metis_client.list_folder(
+                  Etna::Clients::Metis::ListFolderRequest.new(
+                      project_name: project_name,
+                      bucket_name: bucket_name,
+                      folder_path: folder.folder_path,
+                  ),
+              ).files.all.each do |file|
+                matches = matching_expressions
+                          .map { |regex, attribute_name| [regex.match(file.file_path), regex, attribute_name] }
+                          .select { |match, regex, attribute_name| !match.nil? }
+
+                if matches.length > 1
+                  raise "File path #{file.file_path} matches multiple regex, #{matches.map(&:second)}.  Please modify the matching expressions to disambiguate"
+                end
+
+                if matches.length == 1
+                  match, _, attribute_name = matches.first
+                  match_map = match.names.zip(match.captures).to_h
+                  key = [match_map, attribute_name]
+
+                  if attribute_options.dig(attribute_name, :file_collection)
+                    (all_matches[key] ||= []).push(file.file_path)
+                  else
+                    if all_matches.include?(key)
+                      raise "Field #{attribute_name} for #{match_map} identified for two files, #{file.file_path} and #{all_matches[key]}.  Please modify the existing matching expressionts to disambiguate"
+                    end
+
+                    all_matches[key] = [file.file_path]
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        # Subclasses should override this to implement custom logic for how regex matches should match to linking.
+        def matches_to_record_identifiers(match_data)
+          {"project" => project_name}
+        end
+
+        def patient_timepoint_from(word)
+          match = PATIENT_TIMEPOINT_REGEX.match(word)
+          return {} unless match
+
+          return {
+              'patient' => match[1],
+              'timepoint' => "#{match[1]}-#{match[2]}",
+          }
+        end
+
+        def each_revision
+          find_matches.each do |key, file_paths|
+            match_map, attribute_name = key
+            record_identifiers = matches_to_record_identifiers(match_map)
+            id = containing_record_workflow.ensure_record(model_name, record_identifiers)
+            file_paths.each do |file_path|
+              yield [id, revision_for(id, attribute_name, file_path, match_map, record_identifiers)]
+            end
+          end
+        end
+
+        def link_files
+          magma_crud.update_records do |update_request|
+            each_revision do |id, revision|
+              update_request.update_revision(model_name, id, revision)
+            end
+          end
+        end
+
+        def revision_for(id, attribute_name, file_path, match_map, record_identifiers)
+          if attribute_options.dig(attribute_name, :file_collection)
+            file_path = ::File.dirname(file_path)
+            {attribute_name => "https://metis.ucsf.edu/#{project_name}/browse/#{bucket_name}/#{file_path}"}
+          else
+            {attribute_name => {path: "metis://#{project_name}/#{bucket_name}/#{file_path}"}}
+          end
+        end
+
+        def containing_record_workflow
+          @containing_record_workflow ||= EnsureContainingRecordWorkflow.new(magma_crud: magma_crud, models: models)
+        end
+
+        def models
+          @models ||= begin
+            magma_client.retrieve(RetrievalRequest.new(project_name: self.project_name, model_name: 'all')).models
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/etna/clients/magma/workflows/json_converters.rb

@@ -0,0 +1,81 @@
+require 'json'
+
+module Etna
+  module Clients
+    class Magma
+      class ConverterBase
+        def self.convert_attribute_user_json_to_magma_json(user_model_json)
+          json_attribute = Marshal.load(Marshal.dump(user_model_json['attributes']))
+          return unless json_attribute
+
+          json_attribute.keys.each do |attribute_name|
+            json_attribute[attribute_name]['attribute_type'] = Etna::Clients::Magma::AttributeType::IDENTIFIER if user_model_json['identifier'] == attribute_name
+          end
+          json_attribute
+        end
+
+        def self.convert_model_user_json_to_magma_json(model_name, user_json)
+          json_model = Marshal.load(Marshal.dump(user_json))
+          json_model['template'] = {
+            'name' => model_name,
+            'identifier' => user_json['identifier'],
+            'parent' => user_json['parent_model_name'],
+            'attributes' => convert_attribute_user_json_to_magma_json(json_model)
+          }
+          json_model
+        end
+
+        def self.convert_project_user_json_to_magma_json(user_json)
+          magma_models_json = {}
+          user_json['models'].keys.each do |model_name|
+            magma_models_json[model_name] = convert_model_user_json_to_magma_json(
+              model_name,
+              user_json['models'][model_name])
+          end
+          user_json['models'] = magma_models_json
+          user_json
+        end
+
+        def prettify(name)
+          name.split('_').map(&:capitalize).join(' ')
+        end
+      end
+
+      class AttributeActionsConverter < ConverterBase
+        attr_reader :actions
+        def initialize(actions)
+          @actions = JSON.parse(actions.to_json, symbolize_names: true)
+        end
+
+        def camelize(action_name)
+          action_name.split('_').map(&:capitalize).join('')
+        end
+
+        def clazz_name(action_name)
+          "Etna::Clients::Magma::#{camelize(action_name)}Action"
+        end
+
+        def convert
+          actions.map do |action_json|
+            # We use desc and attribute_type to be consistent
+            #   with the other JSON actions...but the
+            #   Magma Model takes type and description.
+            if action_json[:action_name] == 'add_attribute'
+              action_json[:type] = action_json.delete(:attribute_type)
+              action_json[:description] = action_json.delete(:desc)
+            elsif action_json[:action_name] == 'add_link'
+              action_json[:links].first[:type] = action_json[:links].first.delete(:attribute_type)
+              action_json[:links].last[:type] = action_json[:links].last.delete(:attribute_type)
+            end
+
+            clazz = Object.const_get(clazz_name(action_json[:action_name]))
+            clazz.new(**action_json)
+          rescue ArgumentError => e
+            modified_message = "Exception while parsing #{action_json}.\n" + e.message
+            raise ArgumentError.new(modified_message)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/workflows/json_validators.rb

@@ -0,0 +1,447 @@
+require 'json'
+require 'set'
+
+module Etna
+  module Clients
+    class Magma
+      class ValidatorBase
+        attr_reader :errors
+
+        def initialize
+          @errors = []
+        end
+
+        def valid?
+          @errors.length == 0
+        end
+
+        def nil_or_empty?(value)
+          value.nil? || value.empty?
+        end
+
+        def check_key(label, raw, key)
+          @errors << "Missing required key for #{label}: \"#{key}\"." if !raw.dig(key)
+          @errors << "Invalid empty #{key} for #{label}: \"#{raw[key]}\"." if raw.dig(key) && nil_or_empty?(raw[key])
+        end
+
+        def check_in_set(label, raw, key, valid_values)
+          @errors << "Invalid #{key} for #{label}: \"#{raw[key]}\".\nShould be one of #{valid_values}." if raw.dig(key) && !valid_values.include?(raw[key])
+        end
+
+        def name_regex_with_numbers
+          /\A[a-z][a-z0-9]*(_[a-z0-9]+)*\Z/
+        end
+
+        def check_valid_name_with_numbers(label, proposed_name)
+          @errors << "#{label} name \"#{proposed_name}\" must be snake_case and can only consist of letters, numbers, and \"_\"." unless proposed_name =~ name_regex_with_numbers
+        end
+
+        def name_regex_no_numbers
+          /\A[a-z]*(_[a-z]+)*\Z/
+        end
+
+        def model_exists_in_project?(project_magma_models, model_name)
+          !!project_magma_models.model(model_name)
+        end
+
+        def validate!(err_message)
+          @errors = []
+          self.validate
+          raise "#{err_message}\n#{format_errors}" unless valid?
+        end
+
+        def format_errors
+          errors.map { |e| e.gsub("\n", "\n\t") }.join("\n  * ")
+        end
+      end
+
+      class ProjectValidator < ValidatorBase
+        attr_reader :create_args
+
+        def initialize(**create_args)
+          super()
+          @create_args = create_args
+        end
+
+        def validate
+          validate_project_names
+        end
+
+        def validate_project_names
+          check_key('root project', create_args, :project_name)
+          check_key('root project', create_args, :project_name_full)
+          name = create_args[:project_name]
+          @errors << "Project name #{name} must be snake_case and cannot start with a number or \"pg_\"." unless name =~ name_regex_with_numbers && !name.start_with?('pg_')
+        end
+      end
+
+      class RenamesValidator < ValidatorBase
+        attr_reader :models, :renames
+        def initialize(models = Models.new, renames = {})
+          @models = models
+          @renames = renames
+          super()
+        end
+
+        def validate
+          renames.each do |model_name, attribute_renames|
+            attribute_renames.each do |old_name, new_name|
+              keys = @models.build_model(model_name).build_template.build_attributes.attribute_keys
+              if keys.include?(new_name)
+                @errors << "Model #{model_name} trying to rename #{old_name} to #{new_name}, but a different #{new_name} already exists."
+              end
+
+              if !keys.include?(old_name)
+                @errors << "Model #{model_name} trying to rename #{old_name} to #{new_name}, but #{old_name} does not exist."
+              end
+            end
+          end
+        end
+      end
+
+      class AddModelValidator < ValidatorBase
+        attr_reader :model
+
+        def initialize(models = Models.new, model_name = 'project')
+          super()
+          @model = models.build_model(model_name)
+          @models = models
+        end
+
+        def parent_reciprocal_attribute
+          @models.find_reciprocal(model: @model, link_attribute_name: @model.template.parent)
+        end
+
+        def name
+          model&.name
+        end
+
+        def raw
+          model&.template&.raw
+        end
+
+        def is_project?
+          name == 'project'
+        end
+
+        def validate
+          @errors << "Model name #{name} must be snake_case and can only consist of letters and \"_\"." unless name =~ name_regex_no_numbers
+
+          if parent_reciprocal_attribute&.attribute_type != Etna::Clients::Magma::ParentLinkType::TABLE
+            check_key("model #{name}", raw, 'identifier')
+          end
+
+          validate_links
+          validate_attributes
+        end
+
+        def validate_attributes
+          model.template.attributes.attribute_keys.each do |attribute_name|
+            attribute = model.template.attributes.attribute(attribute_name)
+
+            if attribute_name == model.template.identifier
+              attribute_types = [AttributeType::IDENTIFIER]
+            elsif attribute_name == model.template.parent
+              attribute_types = [AttributeType::PARENT]
+            elsif @models.find_reciprocal(model: model, attribute: attribute)&.attribute_type == AttributeType::PARENT
+              attribute_types = AttributeValidator.valid_parent_link_attribute_types
+            else
+              attribute_types = AttributeValidator.valid_add_row_attribute_types
+            end
+
+            attribute_validator = AttributeValidator.new(attribute, attribute_types, @models)
+            attribute_validator.validate
+            @errors += attribute_validator.errors
+          end
+        end
+
+        def validate_links
+          if !is_project? && !@models.model_keys.include?(@model.template.parent)
+            @errors << "Parent model \"#{@model.template.parent}\" for #{name} does not exist in project."
+          end
+
+          if !is_project? && parent_reciprocal_attribute.nil?
+            @errors << "Parent link attributes not defined for model #{name}."
+          end
+
+          link_attributes.each do |attribute|
+            check_key("attribute #{attribute.attribute_name}", attribute.raw, 'link_model_name')
+
+            if attribute.attribute_name != attribute.link_model_name
+              @errors << "Linked model, \"#{attribute.link_model_name}\", does not match attribute #{attribute.attribute_name}, link attribute names must match the model name."
+            end
+
+            unless @models.model_keys.include?(attribute.link_model_name)
+              @errors << "Linked model, \"#{attribute.link_model_name}\", on attribute #{attribute.attribute_name} does not exist!"
+            end
+
+            reciprocal = @models.find_reciprocal(model: @model, attribute: attribute)
+            if reciprocal.nil?
+              @errors << "Linked model, \"#{attribute.link_model_name}\", on attribute #{attribute.attribute_name} does not have a reciprocal link defined."
+            end
+          end
+        end
+
+        def link_attributes
+          @model.template.attributes.all.select do |attribute|
+            attribute.attribute_type == Etna::Clients::Magma::AttributeType::LINK
+          end
+        end
+      end
+
+      class AttributeValidator < ValidatorBase
+        attr_reader :attribute
+
+        def initialize(attribute, valid_attribute_types, project_models)
+          super()
+          @attribute = attribute
+          @valid_attribute_types = valid_attribute_types
+          @valid_validation_types = self.class.valid_validation_types
+          @project_models = project_models
+        end
+
+        def self.valid_add_row_attribute_types
+          Etna::Clients::Magma::AttributeType.entries.reject { |a|
+            a == Etna::Clients::Magma::AttributeType::CHILD ||
+                a == Etna::Clients::Magma::AttributeType::IDENTIFIER ||
+                a == Etna::Clients::Magma::AttributeType::PARENT
+          }.sort
+        end
+
+        def self.valid_parent_link_attribute_types
+          [
+              AttributeType::COLLECTION,
+              AttributeType::TABLE,
+              AttributeType::CHILD,
+          ]
+        end
+
+        def self.valid_update_attribute_types
+          Etna::Clients::Magma::AttributeType.entries.sort
+        end
+
+        def self.valid_validation_types
+          Etna::Clients::Magma::AttributeValidationType.entries.sort
+        end
+
+        def validate
+          validate_basic_attribute_data
+          validate_attribute_validation
+        end
+
+        def validate_basic_attribute_data
+          check_valid_name_with_numbers('Attribute', attribute.attribute_name)
+          check_key("attribute #{attribute.attribute_name}", attribute.raw, 'attribute_type')
+
+          if attribute.link_model_name && ![AttributeType::TABLE, AttributeType::LINK, AttributeType::COLLECTION, AttributeType::PARENT, AttributeType::CHILD].include?(attribute.attribute_type)
+            @errors << "attribute #{attribute.attribute_name} has link_model_name set, but has attribute_type #{attribute.attribute_type}"
+          end
+
+          if attribute.link_model_name && !@project_models.model_keys.include?(attribute.link_model_name)
+            @errors << "attribute #{attribute.attribute_name} has link_model_name value of #{attribute.link_model_name}, but a model by that name does not exist."
+          end
+
+          check_in_set("attribute #{attribute.attribute_name}", attribute.raw, 'attribute_type', @valid_attribute_types)
+        end
+
+        def validate_attribute_validation
+          return unless attribute.validation
+          check_key("attribute #{attribute.attribute_name}, validation", attribute.validation, 'type')
+          check_key("attribute #{attribute.attribute_name}, validation", attribute.validation, 'value')
+          check_in_set("attribute #{attribute.attribute_name}, validation", attribute.validation, 'type', @valid_validation_types)
+        end
+
+        def is_link_attribute?
+          attribute.attribute_type == Etna::Clients::Magma::AttributeType::LINK
+        end
+
+        def is_identifier_attribute?
+          attribute.attribute_type == Etna::Clients::Magma::AttributeType::IDENTIFIER
+        end
+      end
+
+      class AttributeActionValidatorBase < ValidatorBase
+        attr_reader :action, :project_models
+
+        def initialize(action, project_models)
+          super()
+          @action = action
+          @project_models = project_models
+        end
+
+        def action_to_attribute(action)
+          action_json = JSON.parse(action.to_json)
+          # Magma Model uses type and description for Actions,
+          #   but Attribute uses attribute_type and desc.
+          action_json['attribute_type'] = action_json.delete('type')
+          action_json['desc'] = action_json.delete('description')
+          Etna::Clients::Magma::Attribute.new(action_json)
+        end
+
+        def validate
+          raise "Subclasses must implement this method."
+        end
+
+        def exists_in_magma_model?(magma_model_name, attribute_name)
+          !!project_models.model(magma_model_name).template.attributes.attribute(attribute_name)
+        end
+
+        def validate_model_exists(magma_model_name)
+          @errors << "Model \"#{magma_model_name}\" does not exist in project." unless model_exists_in_project?(project_models, magma_model_name)
+        end
+
+        def check_already_exists_in_model(magma_model_name, attribute_name)
+          return unless model_exists_in_project?(project_models, magma_model_name)
+          @errors << "Attribute \"#{attribute_name}\" already exists in model #{magma_model_name}." if exists_in_magma_model?(magma_model_name, attribute_name)
+        end
+
+        def check_does_not_exist_in_model(magma_model_name, attribute_name)
+          return unless model_exists_in_project?(project_models, magma_model_name)
+          @errors << "Attribute \"#{attribute_name}\" does not exist in model #{magma_model_name}." unless exists_in_magma_model?(magma_model_name, attribute_name)
+        end
+      end
+
+      class AddAttributeActionValidator < AttributeActionValidatorBase
+        def initialize(action, project_models)
+          super
+          @attribute = action_to_attribute(action)
+        end
+
+        def validate
+          validate_attribute_data
+          validate_model_exists(action.model_name)
+        end
+
+        def validate_attribute_data
+          validator = AttributeValidator.new(@attribute, AttributeValidator.valid_add_row_attribute_types, project_models)
+          validator.validate
+          @errors += validator.errors unless validator.valid?
+
+          check_already_exists_in_model(action.model_name, action.attribute_name)
+        end
+      end
+
+      class AddLinkActionValidator < AttributeActionValidatorBase
+        def source
+          action.links.first
+        end
+
+        def dest
+          action.links.last
+        end
+
+        def validate
+          validate_links
+          validate_both_models_exist
+          validate_link_data
+        end
+
+        def validate_links
+          check_key("action #{action}", action, :links)
+          @errors << "Must include two link entries, each with \"model_name\", \"attribute_name\", and \"type\"." unless action.links.length == 2
+          check_key("link #{source}", source, :model_name)
+          check_key("link #{source}", source, :attribute_name)
+          check_key("link #{source}", source, :type)
+          check_key("link #{dest}", dest, :model_name)
+          check_key("link #{dest}", dest, :attribute_name)
+          check_key("link #{dest}", dest, :type)
+        end
+
+        def validate_both_models_exist
+          validate_model_exists(source[:model_name])
+          validate_model_exists(dest[:model_name])
+        end
+
+        def validate_link_data
+          # Make sure the attribute names don't already exist in the models,
+          #   and that the types are valid.
+          link_types = Set.new([source[:type], dest[:type]])
+          expected_link_types = Set.new([
+              Etna::Clients::Magma::AttributeType::LINK,
+              Etna::Clients::Magma::AttributeType::COLLECTION
+          ])
+          if link_types != expected_link_types
+            @errors << "You must have one \"link\" and one \"collection\" type in the links."
+          else
+            check_already_exists_in_model(source[:model_name], source[:attribute_name])
+            check_already_exists_in_model(dest[:model_name], dest[:attribute_name])
+
+            @errors << "Links #{source} and #{dest} must point to each other." unless source[:model_name] == dest[:attribute_name] && source[:attribute_name] == dest[:model_name]
+          end
+        end
+      end
+
+      class RenameAttributeActionValidator < AttributeActionValidatorBase
+        def validate
+          validate_action
+          validate_model_exists(action.model_name)
+          validate_proposed_name
+        end
+
+        def validate_action
+          check_key("action #{action}", action, :model_name)
+          check_key("action #{action}", action, :attribute_name)
+          check_key("action #{action}", action, :new_attribute_name)
+        end
+
+        def validate_proposed_name
+          check_does_not_exist_in_model(action.model_name, action.attribute_name)
+          check_valid_name_with_numbers('New attribute', action.new_attribute_name)
+          check_already_exists_in_model(action.model_name, action.new_attribute_name)
+        end
+      end
+
+      class UpdateAttributeActionValidator < AttributeActionValidatorBase
+        def initialize(action, project_models)
+          super
+          @attribute = action_to_attribute(action)
+        end
+
+        def validate
+          validate_attribute_data
+          validate_model_exists(action.model_name)
+        end
+
+        def validate_attribute_data
+          check_does_not_exist_in_model(action.model_name, action.attribute_name)
+        end
+      end
+
+      class AttributeActionsValidator < ValidatorBase
+        attr_reader :actions, :project_models
+
+        def initialize(actions, project_models)
+          super()
+          @actions = actions
+          @project_models = project_models
+        end
+
+        def project_model_names
+          project_models.all.map(&:template).map(&:name)
+        end
+
+        def validate
+          validate_actions
+        end
+
+        def validate_actions
+          actions.each do |action|
+            clazz = Object.const_get(clazz_name(action.action_name))
+            validator = clazz.new(action, project_models)
+            validator.validate
+
+            @errors += validator.errors unless validator.valid?
+          end
+        end
+
+        def camelize(action_name)
+          action_name.split('_').map(&:capitalize).join('')
+        end
+
+        def clazz_name(action_name)
+          "Etna::Clients::Magma::#{camelize(action_name)}ActionValidator"
+        end
+      end
+    end
+  end
+end