etna 0.1.27 → 0.1.32

data/lib/etna.rb

@@ -20,6 +20,7 @@ require_relative './etna/csvs'
 require_relative './etna/environment_scoped'
 require_relative './etna/filesystem'
 require_relative './etna/formatting'
+require_relative './etna/cwl'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/application.rb

@@ -91,6 +91,10 @@ module Etna::Application
     (ENV["#{self.class.name.upcase}_ENV"] || :development).to_sym
   end
 
+  def id
+    ENV["APP_NAME"] || self.class.name.snake_case.split(/::/).last
+  end
+
   def find_descendents(klass)
     ObjectSpace.each_object(Class).select do |k|
       k < klass

data/lib/etna/client.rb

@@ -17,6 +17,29 @@ module Etna
 
     attr_reader :routes
 
+    def signed_route_path(route, params)
+      path = route_path(route,params)
+
+      signatory = params.delete(:signatory)
+
+      return path unless signatory
+
+      hmac = Etna::Hmac.new(
+        signatory,
+        method: route[:method],
+        host: URI(@host).host,
+        path: path,
+        expiration: (DateTime.now + 10).iso8601,
+        id: signatory.id,
+        nonce: SecureRandom.hex,
+        headers: params.except(*route[:params].map(&:to_sym))
+      )
+
+      url_params = hmac.url_params(route[:method] == 'GET')
+
+      return url_params[:path] + '?' + url_params[:query]
+    end
+
     def route_path(route, params)
       Etna::Route.path(route[:route], params)
     end
@@ -60,14 +83,19 @@ module Etna
       @routes.each do |route|
         next unless route[:name]
         self.define_singleton_method(route[:name]) do |params = {}|
-
           missing_params = (route[:params] - params.keys.map(&:to_s))
+
           unless missing_params.empty?
             raise ArgumentError, "Missing required #{missing_params.size > 1 ?
                 'params' : 'param'} #{missing_params.join(', ')}"
           end
 
-          response = send(route[:method].downcase, route_path(route, params), params)
+          response = send(
+            route[:method].downcase,
+            signed_route_path(route, params),
+            params
+          )
+
           if block_given?
             yield response
           else

data/lib/etna/clients/magma/formatting.rb

@@ -1 +1,2 @@
-require_relative 'formatting/models_csv'
+require_relative 'formatting/models_csv'
+require_relative 'formatting/models_odm_xml'

data/lib/etna/clients/magma/formatting/models_odm_xml.rb

@@ -0,0 +1,293 @@
+require 'nokogiri'
+
+module Etna
+  module Clients
+    class Magma
+      module ModelsOdmXml
+      end
+    end
+  end
+end
+
+module Etna
+  module Clients
+    class Magma
+      module ModelsOdmXml
+        module Prettify
+          def prettify(name)
+            name.split('_').map(&:capitalize).join(' ')
+          end
+
+          def shorten(name)
+            name.gsub('_', '').capitalize
+          end
+        end
+
+        class Exporter
+          include Prettify
+
+          attr_reader :project_name, :models
+
+          def initialize(project_name:, models:)
+            @project_name = project_name
+            @models = models
+          end
+
+          def data_type_map
+            @data_type_map ||= begin
+              map = {}
+              map[Etna::Clients::Magma::AttributeType::STRING] = 'text'
+              map[Etna::Clients::Magma::AttributeType::DATE_TIME] = 'date'
+              map[Etna::Clients::Magma::AttributeType::BOOLEAN] = 'text'
+              map[Etna::Clients::Magma::AttributeType::FLOAT] = 'float'
+              map[Etna::Clients::Magma::AttributeType::INTEGER] = 'integer'
+
+              map
+            end
+          end
+
+          def redcap_field_type_map
+            @redcap_field_type_map ||= begin
+              map = {}
+              map[Etna::Clients::Magma::AttributeType::STRING] = 'textarea'
+              map[Etna::Clients::Magma::AttributeType::DATE_TIME] = 'text'
+              map[Etna::Clients::Magma::AttributeType::BOOLEAN] = 'radio'
+              map[Etna::Clients::Magma::AttributeType::FLOAT] = 'text'
+              map[Etna::Clients::Magma::AttributeType::INTEGER] = 'text'
+
+              map
+            end
+          end
+
+          def redcap_text_validation_map
+            @redcap_text_validation_map ||= begin
+              map = {}
+              map[Etna::Clients::Magma::AttributeType::DATE_TIME] = 'date_mdy'
+              map[Etna::Clients::Magma::AttributeType::FLOAT] = 'float'
+              map[Etna::Clients::Magma::AttributeType::INTEGER] = 'int'
+
+              map
+            end
+          end
+
+          def write_models(output_io: nil, filename: nil)
+            @document = Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+              xml.ODM(odm_headers) do
+                xml.Study(OID: "Project.#{shorten(project_name)}") do
+                end
+                global_variables(xml)
+                metadata(xml)
+              end
+            end
+
+            @document.to_xml
+          end
+
+          def odm_headers
+            {
+              xmlns: "http://www.cdisc.org/ns/odm/v1.3",
+              'xmlns:ds': "http://www.w3.org/2000/09/xmldsig#",
+              'xmlns:xsi': "http://www.w3.org/2001/XMLSchema-instance",
+              'xmlns:redcap': "https://projectredcap.org",
+              'xsi:schemaLocation': "http://www.cdisc.org/ns/odm/v1.3 schema/odm/ODM1-3-1.xsd",
+              ODMVersion: "1.3.2",
+              FileOID: "000-00-0000",
+              FileType: "Snapshot",
+              Description: project_name,
+              AsOfDateTime: DateTime.now,
+              CreationDateTime: DateTime.now,
+              SourceSystem: "Magma",
+              SourceSystemVersion: DateTime.now
+            }
+          end
+
+          def global_variables(xml)
+            # Includes general metadata about the project, as well as
+            # declarations of all repeating instruments,
+            #   which seem like Timepoints to me.
+            # NOTE:
+            # <redcap:Purpose>0</redcap:Purpose>
+            # 0 = Practice / just for fun
+            # 1 = Operational Support
+            # 2 = Research
+            # 3 = Quality Improvement
+            # 4 = Other
+
+            xml.GlobalVariables do
+              xml.StudyName "#{project_name}"
+              xml.StudyDescription "#{project_name} - Data Library integration project"
+              xml.ProtocolName "#{project_name}"
+              xml.send('redcap:RecordAutonumberingEnabled', 1)
+              xml.send('redcap:CustomRecordLabel')
+              xml.send('redcap:SecondaryUniqueField')
+	            xml.send('redcap:SchedulingEnabled', 0)
+	            xml.send('redcap:SurveysEnabled', 0)
+              xml.send('redcap:SurveyInvitationEmailField')
+              xml.send('redcap:Purpose', 2) # 2 == research
+              xml.send('redcap:PurposeOther')
+              xml.send('redcap:ProjectNotes', "Used to easily ingest clinical data for #{project_name} into the Data Library.")
+              xml.send('redcap:MissingDataCodes')
+
+              repeating_instruments(xml)
+            end
+          end
+
+          def repeating_attribute_types
+            # We don't have a good indicator for what is a repeating
+            #   attribute for REDCap...
+            # Will this work for models without timepoint, that go
+            #   straight from subject -> sample?
+            models.model('subject').template.attributes.all.select do |attribute|
+              Etna::Clients::Magma::AttributeType::COLLECTION == attribute.attribute_type
+            end
+          end
+
+          def clinical_dictionaries
+            # Our indicator for if something needs a REDCap form will be any
+            #   model with a dictionary.
+            models.all.select do |model|
+              model.template.raw['dictionary']
+            end.map do |model|
+              model.template.dictionary
+            end
+          end
+
+          def repeating_instruments(xml)
+            # Now we get into repeating instruments and events.
+            # From a Magma model perspective, this should be
+            #   Timepoint that hangs off of
+            #   a Subject model.
+            xml.send('redcap:RepeatingInstrumentsAndEvents') do
+              repeating_attribute_types.map do |repeating_attribute|
+                write_repeating_instrument_xml(xml, repeating_attribute)
+              end
+            end
+          end
+
+          def write_repeating_instrument_xml(xml, instrument)
+            node = xml.send('redcap:RepeatingInstrument')
+            node['redcap:UniqueEventName'] = 'event_1_arm_1'
+            node['redcap:RepeatInstrument'] = instrument.attribute_name
+            node['redcap:CustomLabel'] = instrument.display_name || instrument.attribute_name.capitalize
+
+            node
+          end
+
+          def metadata(xml)
+            # Includes form and field definitions
+            xml.MetaDataVersion(
+              OID: "Metadata.#{shorten(project_name)}_#{DateTime.now}",
+              Name: project_name,
+              'redcap:RecordIdField': 'record_id'
+            ) do
+              clinical_dictionaries.map do |dictionary|
+                # Each Magma dictionary needs a FormDef, with
+                #   ItemGroupRef children for each form page (?).
+                # Each ItemGroupRef requires a corresponding ItemGroupDef
+                #   with ItemRef children for each input (?).
+                # Each ItemRef requires a correspdonding ItemDef
+                #   that defines the label and type, and includes
+                #   <Question> as a label (?).
+                # Option validations are present as a CodeList (with
+                #   CodeListItem children).
+                write_form_def(xml, dictionary)
+                write_item_group_def(xml, dictionary)
+                write_item_def(xml, dictionary)
+                write_code_list(xml, dictionary)
+              end
+            end
+          end
+
+          def write_form_def(xml, dictionary)
+            xml.FormDef(
+              OID: "Form.#{dictionary.model_name}",
+              Name: dictionary.model_name.capitalize,
+              Repeating: "No",
+              'redcap:FormName': dictionary.model_name
+            ) do
+              # Assume a single item group
+              xml.ItemGroupRef(
+                ItemGroupOID: "#{dictionary.model_name}.attributes",
+                Mandatory: "No"
+              ) do
+              end
+            end
+          end
+
+          def write_item_group_def(xml, dictionary)
+            xml.ItemGroupDef(
+              OID: "#{dictionary.model_name}.attributes",
+              Name: "#{dictionary.model_name.capitalize} Attributes",
+              Repeating: "No"
+            ) do
+              dictionary.attributes.keys.map do |attribute_name|
+                xml.ItemRef(
+                  ItemOID: "#{dictionary.model_name}.#{attribute_name}", # Does this need to be unique across all items?
+                  Mandatory: "No",
+                  'redcap:Variable': attribute_name # Does this need to be unique?
+                ) do
+                end
+              end
+            end
+          end
+
+          def write_item_def(xml, dictionary)
+            model_attributes = models.model(dictionary.model_name).template.attributes
+            dictionary.attributes.keys.map do |attribute_name|
+              attribute = model_attributes.attribute(attribute_name)
+              attribute_type = attribute.attribute_type
+              params = {
+                OID: "#{dictionary.model_name}.#{attribute_name}", # Does this need to be unique across all items?
+                Name: attribute_name,
+                DataType: data_type_map[attribute_type] || 'text',
+                'redcap:Variable': attribute_name,
+                'redcap:FieldType': redcap_field_type_map[attribute_type] || 'text',
+                Length: '999' # How could we infer shorter values?
+              }
+
+              params['redcap:TextValidationType'] = redcap_text_validation_map[attribute_type] if redcap_text_validation_map[attribute_type]
+              params['redcap:FieldNote'] = attribute.desc if attribute.desc
+              xml.ItemDef(params) do
+                xml.Question do
+                  xml.send('TranslatedText', attribute_name.capitalize)
+                end
+
+                if attribute.validation && Etna::Clients::Magma::AttributeValidationType::ARRAY == attribute.validation['type']
+                  xml.CodeListRef(
+                    CodeListOID: "#{dictionary.model_name}.#{attribute_name}.choices"
+                  ) do
+                  end
+                end
+              end
+            end
+          end
+
+          def write_code_list(xml, dictionary)
+            model_attributes = models.model(dictionary.model_name).template.attributes
+            dictionary.attributes.keys.map do |attribute_name|
+              attribute = model_attributes.attribute(attribute_name)
+              if attribute.validation && Etna::Clients::Magma::AttributeValidationType::ARRAY == attribute.validation['type']
+                xml.CodeList(
+                  OID: "#{dictionary.model_name}.#{attribute_name}.choices",
+                  Name: "#{attribute_name}",
+                  DataType: "text",
+                  'redcap:Variable': attribute_name
+                ) do
+                  attribute.validation['value'].map do |option|
+                    xml.CodeListItem(
+                      CodedValue: option
+                    ) do
+                      xml.Decode() do
+                        xml.send('TranslatedText', option)
+                      end
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/etna/clients/magma/models.rb

@@ -519,6 +519,18 @@ module Etna
           @raw['desc'] = val
         end
 
+        # description and description= are needed
+        #   to make UpdateAttribute actions
+        #   work in the model_synchronization_workflow for
+        #   desc.
+        def description
+          raw['desc']
+        end
+
+        def description=(val)
+          @raw['desc'] = val
+        end
+
         def display_name
           raw['display_name']
         end
@@ -589,7 +601,7 @@ module Etna
         COPYABLE_ATTRIBUTE_ATTRIBUTES = [
             :attribute_name, :attribute_type, :desc, :display_name, :format_hint,
             :hidden, :link_model_name, :read_only, :attribute_group, :unique, :validation,
-            :restricted
+            :restricted, :description
         ]
 
         EDITABLE_ATTRIBUTE_ATTRIBUTES = UpdateAttributeAction.members & COPYABLE_ATTRIBUTE_ATTRIBUTES

data/lib/etna/clients/magma/workflows/create_project_workflow.rb

@@ -70,7 +70,7 @@ module Etna
           puts "Creating Janus project."
           create_janus_project!
           puts "Done! Adding you as an administrator on the project."
-          add_janus_user(user['email'], "#{user['first']} #{user['last']}", 'editor')
+          add_janus_user(user['email'], "#{user['name']}", 'editor')
           promote_to_administrator(user['email'])
           update_magma_client_token!
 

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -36,11 +36,28 @@ module Etna
           documents = Documents.new({})
           last_page = nil
           while last_page.nil? || last_page.models.model_keys.map { |k| last_page.models.model(k).documents.raw.length }.sum > 0
+            attempts = 0
             begin
+              attempts += 1
               last_page = magma_client.retrieve(request)
+            # Unfortunately, paging in magma is not great and times out from time to time.
+            rescue Net::ReadTimeout => e
+              if attempts > 5
+                raise e
+              end
+
+              retry
             rescue Etna::Error => e
-              raise e unless e.message.include?('not found')
-              break
+              if e.status === 502
+                if attempts > 5
+                  raise e
+                end
+
+                retry
+              else
+                raise e unless e.message.include?('not found')
+                break
+              end
             end
 
             documents += last_page.models.model(model_name).documents unless block_given?

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -1,5 +1,4 @@
 require 'ostruct'
-require 'digest'
 require 'fileutils'
 require 'tempfile'
 
@@ -9,11 +8,11 @@ module Etna
       class MaterializeDataWorkflow < Struct.new(
           :metis_client, :magma_client, :project_name,
           :model_name, :model_filters, :model_attributes_mask,
-          :filesystem, :logger, :stub_files,
-          :skip_tmpdir, keyword_init: true)
+          :filesystem, :logger, :stub_files, :concurrency,
+          :record_names, keyword_init: true)
 
         def initialize(**kwds)
-          super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
+          super(**({filesystem: Etna::Filesystem.new, concurrency: 10, record_names: "all"}.update(kwds)))
         end
 
         def magma_crud
@@ -25,31 +24,47 @@ module Etna
         end
 
         def materialize_all(dest)
-          tmpdir = skip_tmpdir ? nil : filesystem.tmpdir
+          templates = {}
+
+          semaphore = Concurrent::Semaphore.new(concurrency)
+          errors = Queue.new
+
+          model_walker.walk_from(
+              model_name,
+              record_names,
+              model_attributes_mask: model_attributes_mask,
+              model_filters: model_filters,
+              page_size: 20,
+          ) do |template, document|
+            logger&.info("Materializing #{template.name}##{document[template.identifier]}")
+            templates[template.name] = template
+
+            begin
+              if (error = errors.pop(true))
+                raise error
+              end
+            rescue ThreadError
+            end
 
-          begin
-            model_walker.walk_from(
-                model_name,
-                model_attributes_mask: model_attributes_mask,
-                model_filters: model_filters,
-            ) do |template, document|
-              logger&.info("Materializing #{template.name}##{document[template.identifier]}")
-              materialize_record(dest, tmpdir, template, document)
+            semaphore.acquire
+            Thread.new do
+              begin
+                materialize_record(dest, template, document)
+              rescue => e
+                errors << e
+              ensure
+                semaphore.release
+              end
             end
-          ensure
-            filesystem.rm_rf(tmpdir) unless skip_tmpdir
           end
-        end
 
-        def each_root_record
-          request = RetrievalRequest.new(project_name: project_name, model_name: model_name, record_names: "all",
-              filter: filter, page_size: 100, page: 1)
-          magma_crud.page_records(model_name, request) do |response|
-            model = response.models.model(model_name)
-            template = model.template
-            model.documents.document_keys.each do |key|
-              yield template, model.documents.document(key)
+          semaphore.acquire(concurrency)
+
+          begin
+            if (error = errors.pop(true))
+              raise error
             end
+          rescue ThreadError
           end
         end
 
@@ -78,11 +93,10 @@ module Etna
           @sync_metis_data_workflow ||= Etna::Clients::Metis::SyncMetisDataWorkflow.new(
               metis_client: metis_client,
               logger: logger,
-              skip_tmpdir: skip_tmpdir,
               filesystem: filesystem)
         end
 
-        def materialize_record(dest_dir, tmpdir, template, record)
+        def materialize_record(dest_dir, template, record)
           record_to_serialize = record.dup
 
           each_file(template, record) do |attr_name, url, filename, idx|
@@ -91,14 +105,16 @@ module Etna
             end
 
             dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: "_#{attr_name}_#{idx}#{File.extname(filename)}"))
-            sync_metis_data_workflow.copy_file(bin_root_dir: dest_dir, tmpdir: tmpdir, dest: dest_file, url: url, stub: stub_files)
-            record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
+            sync_metis_data_workflow.copy_file(dest: dest_file, url: url, stub: stub_files)
+            record_to_serialize[attr_name] << {file: dest_file, original_filename: filename}
           end
 
           dest_file = File.join(dest_dir, metadata_file_name(record_name: record[template.identifier], record_model_name: template.name, ext: '.json'))
           filesystem.mkdir_p(File.dirname(dest_file))
-          filesystem.with_writeable(dest_file, "w") do |io|
-            io.write(record_to_serialize.to_json)
+          json = record_to_serialize.to_json
+
+          filesystem.with_writeable(dest_file, "w", size_hint: json.bytes.length) do |io|
+            io.write(json)
           end
         end