etna 0.1.20 → 0.1.25

data/lib/commands.rb

@@ -54,7 +54,7 @@ class EtnaApp
       boolean_flags << '--ignore-ssl'
 
       def execute(host, ignore_ssl: false)
-        polyphemus_client ||= Etna::Clients::Polyphemus.new(
+        polyphemus_client = Etna::Clients::Polyphemus.new(
             host: host,
             token: token(ignore_environment: true),
             ignore_ssl: ignore_ssl)
@@ -62,7 +62,7 @@ class EtnaApp
             polyphemus_client: polyphemus_client,
             config_file: EtnaApp.config_file_path)
         config = workflow.update_configuration_file(ignore_ssl: ignore_ssl)
-        logger.info("Updated #{config.environment} configuration from #{host}.")
+        logger&.info("Updated #{config.environment} configuration from #{host}.")
       end
 
       def setup(config)
@@ -226,7 +226,7 @@ class EtnaApp
 
           @last_load = File.stat(file).mtime
           @changeset = File.open(file, 'r') do |f|
-            workflow.prepare_changeset_from_csv(f) do |err|
+            workflow.prepare_changeset_from_csv(io: f) do |err|
               @errors << err
             end
           end
@@ -299,6 +299,85 @@ class EtnaApp
             workflow.write_csv_io(filename: file)
           end
         end
+
+        class LoadTableFromCsv < Etna::Command
+          include WithEtnaClients
+
+          boolean_flags << '--execute'
+
+          def execute(project_name, model_name, file_path, execute: false)
+            request = Etna::Clients::Magma::RetrievalRequest.new(project_name: project_name)
+            request.model_name = model_name
+            request.attribute_names = 'all'
+            request.record_names = 'all'
+            model = magma_client.retrieve(request).models.model(model_name)
+            model_parent_name = model.template.attributes.all.select do |attribute|
+              attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT
+            end.first.name
+
+            other_attribute_names = model.template.attributes.all.reject do |attribute|
+              attribute.attribute_type == Etna::Clients::Magma::AttributeType::PARENT
+            end.map do |attribute|
+              attribute.name
+            end
+
+            # NOTE: This does not call ensure_parent currently because of MVIR1 consent--
+            #   if the timepoint doesn't exist, the patient may be no study? (one example, at least)
+            update_request = Etna::Clients::Magma::UpdateRequest.new(project_name: project_name)
+
+            data = CSV.parse(File.read(file_path), headers: true)
+
+            data.by_row.each do |row|
+              revision = {}
+              other_attribute_names.each do |attribute_name|
+                revision[attribute_name] = row[attribute_name] unless row[attribute_name].nil?
+              end
+              update_request.append_table(model_parent_name, row[model_parent_name], model_name, revision)
+            end
+
+            puts update_request
+
+            if execute
+              magma_client.update_json(update_request)
+            end
+          end
+        end
+      end
+
+      class LoadFromRedcap < Etna::Command
+        include WithEtnaClients
+        include WithLogger
+        include StrongConfirmation
+
+        boolean_flags << '--commit'
+        string_flags << '--models'
+
+        def execute(project_name, redcap_tokens, models: "all", commit: false)
+          raise "Must provide at least one REDCap token (comma-separated)." unless redcap_tokens.split(',').length > 0
+
+          puts "NOTE: This is a **preview** of what the data loading will look like. Use the --commit flag to load records into Magma." unless commit
+
+          polyphemus_client.job(Etna::Clients::Polyphemus::RedcapJobRequest.new(
+            model_names: "all" == models ? "all" : models.split(','),
+            redcap_tokens: redcap_tokens.split(','),
+            project_name: project_name,
+            commit: commit
+          )) do |response|
+            response.read_body do |chunk|
+              puts clean_sne_message(chunk)
+            end
+          end
+        end
+
+        def clean_sne_message(chunk)
+          chunk.split("\n").reject do |c|
+            c.start_with?("retry:") || c.start_with?("event:")
+          end.map do |c|
+            c.gsub("data:", "").strip
+          end.reject do |c|
+            c.empty?
+          end
+        end
       end
     end
   end

data/lib/etna.rb

@@ -18,6 +18,7 @@ require_relative './etna/spec'
 require_relative './etna/clients'
 require_relative './etna/csvs'
 require_relative './etna/environment_scoped'
+require_relative './etna/filesystem'
 
 class EtnaApp
   include Etna::Application

data/lib/etna/client.rb

@@ -1,14 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require 'rack/utils'
 
 module Etna
   class Client
-    def initialize(host, token, routes_available: true, persistent: true, ignore_ssl: false)
+    def initialize(host, token, routes_available: true, ignore_ssl: false)
       @host = host.sub(%r!/$!, '')
       @token = token
-      @persistent = persistent
       @ignore_ssl = ignore_ssl
 
       if routes_available
@@ -38,6 +36,10 @@ module Etna
       query_request(Net::HTTP::Get, endpoint, params, &block)
     end
 
+    def head(endpoint, params = {}, &block)
+      query_request(Net::HTTP::Head, endpoint, params, &block)
+    end
+
     def options(endpoint, params = {}, &block)
       query_request(Net::HTTP::Options, endpoint, params, &block)
     end
@@ -79,16 +81,6 @@ module Etna
       end
     end
 
-    def persistent_connection
-      @http ||= begin
-        http = Net::HTTP::Persistent.new
-        http.read_timeout = 3600
-        http.verify_mode = OpenSSL::SSL::VERIFY_NONE if @ignore_ssl
-        http
-      end
-    end
-
-
     def body_request(type, endpoint, params = {}, &block)
       uri = request_uri(endpoint)
       req = type.new(uri.request_uri, request_params)
@@ -141,36 +133,23 @@ module Etna
 
     def request(uri, data)
       if block_given?
-        if @persistent
-          persistent_connection.request(uri, data) do |response|
+        verify_mode = @ignore_ssl ?
+          OpenSSL::SSL::VERIFY_NONE :
+          OpenSSL::SSL::VERIFY_PEER
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+          http.request(data) do |response|
             status_check!(response)
             yield response
           end
-        else
-          verify_mode = @ignore_ssl ?
-            OpenSSL::SSL::VERIFY_NONE :
-            OpenSSL::SSL::VERIFY_PEER
-          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
-            http.request(data) do |response|
-              status_check!(response)
-              yield response
-            end
-          end
         end
       else
-        if @persistent
-          response = persistent_connection.request(uri, data)
+        verify_mode = @ignore_ssl ?
+          OpenSSL::SSL::VERIFY_NONE :
+          OpenSSL::SSL::VERIFY_PEER
+        Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
+          response = http.request(data)
           status_check!(response)
           return response
-        else
-          verify_mode = @ignore_ssl ?
-            OpenSSL::SSL::VERIFY_NONE :
-            OpenSSL::SSL::VERIFY_PEER
-          Net::HTTP.start(uri.host, uri.port, use_ssl: true, verify_mode: verify_mode) do |http|
-            response = http.request(data)
-            status_check!(response)
-            return response
-          end
         end
       end
     end

data/lib/etna/clients/base_client.rb

@@ -0,0 +1,39 @@
+require 'base64'
+require 'json'
+require 'date'
+
+module Etna
+  module Clients
+    class BaseClient
+      attr_reader :host, :token, :ignore_ssl
+      def initialize(host:, token:, ignore_ssl: false)
+        raise "#{self.class.name} client configuration is missing host." unless host
+        raise "#{self.class.name} client configuration is missing token." unless token
+
+        @token = token
+        raise "Your token is expired." if token_expired?
+
+        @etna_client = ::Etna::Client.new(
+          host,
+          token,
+          routes_available: false,
+          ignore_ssl: ignore_ssl)
+        @host = host
+        @ignore_ssl = ignore_ssl
+      end
+
+      def token_expired?
+        # Has the token already expired?
+        token_will_expire?(0)
+      end
+
+      def token_will_expire?(offset=3000)
+        # offset in seconds
+        # Will the user's token expire in the given amount of time?
+        epoch_seconds = JSON.parse(Base64.urlsafe_decode64(token.split('.')[1]))["exp"]
+        expiration = DateTime.strptime(epoch_seconds.to_s, "%s")
+        expiration <= DateTime.now.new_offset + offset
+      end
+    end
+  end
+end

data/lib/etna/clients/janus/client.rb

@@ -1,23 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
 require_relative '../../client'
 require_relative './models'
+require_relative '../base_client'
 
 module Etna
   module Clients
-    class Janus
-      def initialize(host:, token:, persistent: true, ignore_ssl: false)
-        raise 'Janus client configuration is missing host.' unless host
-        raise 'Janus client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          routes_available: false,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
-      end
-
+    class Janus < Etna::Clients::BaseClient
       def get_project(get_project_request = GetProjectRequest.new)
         html = nil
         @etna_client.get(

data/lib/etna/clients/janus/models.rb

@@ -1,11 +1,12 @@
 require 'ostruct'
 require_relative '../../json_serializable_struct'
+require_relative '../base_client'
 
 # TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
 # common definitions.  For nowe I've written them out by hand here.
 module Etna
   module Clients
-    class Janus
+    class Janus < Etna::Clients::BaseClient
       class GetProjectRequest < Struct.new(:project_name, keyword_init: true)
         include JsonSerializableStruct
 

data/lib/etna/clients/magma/client.rb

@@ -1,27 +1,12 @@
-require 'net/http/persistent'
 require 'net/http/post/multipart'
 require 'singleton'
+require_relative '../base_client'
 require_relative '../../client'
 require_relative './models'
 
 module Etna
   module Clients
-    class Magma
-      attr_reader :host, :token, :ignore_ssl
-      def initialize(host:, token:, persistent: true, ignore_ssl: false)
-        raise 'Magma client configuration is missing host.' unless host
-        raise 'Magma client configuration is missing token.' unless token
-        @etna_client = ::Etna::Client.new(
-          host,
-          token,
-          routes_available: false,
-          persistent: persistent,
-          ignore_ssl: ignore_ssl)
-        @host = host
-        @token = token
-        @ignore_ssl = ignore_ssl
-      end
-
+    class Magma < Etna::Clients::BaseClient
       # This endpoint returns models and records by name:
       # e.g. params:
       # {

data/lib/etna/clients/magma/models.rb

@@ -3,12 +3,13 @@ require_relative '../../json_serializable_struct'
 require_relative '../../multipart_serializable_nested_hash'
 require_relative '../../directed_graph'
 require_relative '../enum'
+require_relative '../base_client'
 
 # TODO:  In the near future, I'd like to transition to specifying apis via SWAGGER and generating model stubs from the
 # common definitions.  For nowe I've written them out by hand here.
 module Etna
   module Clients
-    class Magma
+    class Magma < Etna::Clients::BaseClient
       class RetrievalRequest < Struct.new(:model_name, :attribute_names, :record_names, :project_name, :page, :page_size, :order, :filter, keyword_init: true)
         include JsonSerializableStruct
 
@@ -316,8 +317,11 @@ module Etna
         end
 
         def document(document_key)
-          return nil unless raw.include?(document_key)
-          raw[document_key]
+          if document_key.is_a?(String)
+            raw[document_key]
+          else
+            raw[document_key&.to_s]
+          end
         end
       end
 

data/lib/etna/clients/magma/workflows.rb

@@ -8,3 +8,5 @@ require_relative './workflows/update_attributes_from_csv_workflow'
 require_relative './workflows/create_project_workflow'
 require_relative './workflows/add_project_models_workflow'
 require_relative './workflows/attribute_actions_from_json_workflow'
+require_relative './workflows/materialize_magma_record_files_workflow'
+require_relative './workflows/walk_model_tree_workflow'

data/lib/etna/clients/magma/workflows/crud_workflow.rb

@@ -31,13 +31,19 @@ module Etna
                 record_names: 'all',
                 page_size: 20,
                 page: 1,
-            ))
+            ), &block)
 
           documents = Documents.new({})
           last_page = nil
-          while last_page.nil? || last_page.raw.length >= 20
-            last_page = magma_client.retrieve(request).models.model(model_name).documents
-            documents += last_page unless block_given?
+          while last_page.nil? || last_page.models.model_keys.map { |k| last_page.models.model(k).documents.raw.length }.sum > 0
+            begin
+              last_page = magma_client.retrieve(request)
+            rescue Etna::Error => e
+              raise e unless e.message.include?('not found')
+              break
+            end
+
+            documents += last_page.models.model(model_name).documents unless block_given?
             yield last_page if block_given?
             request.page += 1
           end

data/lib/etna/clients/magma/workflows/materialize_magma_record_files_workflow.rb

@@ -0,0 +1,135 @@
+require 'ostruct'
+require 'digest'
+require 'fileutils'
+require 'tempfile'
+
+module Etna
+  module Clients
+    class Magma
+      class MaterializeDataWorkflow < Struct.new(
+          :metis_client, :magma_client, :project_name,
+          :model_name, :model_filters, :model_attributes_mask,
+          :filesystem, :logger, :stub_files,
+          keyword_init: true)
+
+        def initialize(**kwds)
+          super(**({filesystem: Etna::Filesystem.new}.update(kwds)))
+        end
+
+        def magma_crud
+          @magma_crud ||= Etna::Clients::Magma::MagmaCrudWorkflow.new(magma_client: magma_client, project_name: project_name)
+        end
+
+        def model_walker
+          @model_walker ||= WalkModelTreeWorkflow.new(magma_crud: magma_crud, logger: logger)
+        end
+
+        def with_materialized_dir(&block)
+          tmp_dir = filesystem.tmpdir
+
+          begin
+            model_walker.walk_from(
+                model_name,
+                model_attributes_mask: model_attributes_mask,
+                model_filters: model_filters,
+            ) do |template, document|
+              logger&.info("Materializing #{template.name}##{document[template.identifier]}")
+              materialize_record(tmp_dir, template, document)
+            end
+
+            yield tmp_dir
+          ensure
+            filesystem.rm_rf(tmp_dir)
+          end
+        end
+
+        def each_root_record
+          request = RetrievalRequest.new(project_name: project_name, model_name: model_name, record_names: "all",
+              filter: filter, page_size: 100, page: 1)
+          magma_crud.page_records(model_name, request) do |response|
+            model = response.models.model(model_name)
+            template = model.template
+            model.documents.document_keys.each do |key|
+              yield template, model.documents.document(key)
+            end
+          end
+        end
+
+        def each_file(template, record, &block)
+          results = []
+
+          template.attributes.all.each do |attribute|
+            if attribute.attribute_type == AttributeType::FILE_COLLECTION
+              record[attribute.name]&.each_with_index do |file, i|
+                results << [attribute, file, i]
+              end
+            elsif attribute.attribute_type == AttributeType::FILE
+              results << [attribute, record[attribute.name], 0]
+            end
+          end
+
+          results.each do |attr, file, idx|
+            next if file.nil?
+            next unless file.is_a?(Hash)
+            next unless file['url']
+            yield attr.name, file['url'], (file['original_filename'] || File.basename(file['path'])), idx
+          end
+        end
+
+        def materialize_record(dest_dir, template, record)
+          record_to_serialize = record.dup
+          metadata_path = metadata_file_name(record_name: record[template.identifier], record_model_name: template.name)
+
+          each_file(template, record) do |attr_name, url, filename, idx|
+            metadata = metis_client.file_metadata(url)
+            etag = metadata[:etag]
+            size = metadata[:size]
+
+            if idx == 0
+              record_to_serialize[attr_name] = []
+            end
+
+            dest_file = bin_file_name(etag: etag)
+            record_to_serialize[attr_name] << { file: dest_file, original_filename: filename }
+
+            # Already materialized, continue
+            if filesystem.exist?(dest_file)
+              next
+            end
+
+            logger&.info("materializing file #{filename} (#{size} bytes)")
+            filesystem.mkdir_p(File.dirname(File.join(dest_dir, dest_file)))
+
+            filesystem.with_writeable(File.join(dest_dir, dest_file), "w") do |io|
+              if stub_files
+                io.write("(stub) #{filename}: #{size} bytes")
+              else
+                metis_client.download_file(url) do |chunk|
+                  if Random.rand < 0.1
+                    logger&.info("Writing #{chunk.length} bytes into #{dest_file}")
+                  end
+
+                  io.write(chunk)
+                end
+              end
+            end
+          end
+
+          dest_file = File.join(dest_dir, metadata_path)
+          filesystem.mkdir_p(File.dirname(dest_file))
+          filesystem.with_writeable(dest_file, "w") do |io|
+            io.write(record_to_serialize.to_json)
+          end
+        end
+
+        def metadata_file_name(record_name:, record_model_name:)
+          "#{record_model_name}/#{record_name.gsub(/\s/, '_')}.json"
+        end
+
+        def bin_file_name(etag:)
+          "bin/#{etag}"
+        end
+      end
+    end
+  end
+end